Commit e646f26c authored by Daniel Wolf's avatar Daniel Wolf
Browse files

Add filter to query log

This commit also adds various optimizations to the DnsQuery entity:
 - responses is not a MutableList anymore. This allows emptyList() to be used, saving memory.
 (- because of the prior point the query listener does not need to convert the responses to a MutableList from an immutable list, saving memory)
 - parsedResponses is now lazy. Before it was calculated for every row in the query log (every time it was scrolled to)
 - Added isHostBlockedByDnsServer, which makes it obsolete for the query log to calculate this flag for every row.
 - The StringListConverter now doesn't call gson for empty lists, saving time
 -

Closes #261
parent 8e8b21a1
{
"formatVersion": 1,
"database": {
"version": 12,
"identityHash": "4b8a3541fc27086145381f661dbb9a39",
"entities": [
{
"tableName": "CachedResponse",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`dnsName` TEXT NOT NULL, `type` INTEGER NOT NULL, `records` TEXT NOT NULL, PRIMARY KEY(`dnsName`, `type`))",
"fields": [
{
"fieldPath": "dnsName",
"columnName": "dnsName",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "records",
"columnName": "records",
"affinity": "TEXT",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"dnsName",
"type"
],
"autoGenerate": false
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "DnsQuery",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `type` INTEGER NOT NULL, `name` TEXT NOT NULL, `askedServer` TEXT, `responseSource` TEXT NOT NULL, `questionTime` INTEGER NOT NULL, `responseTime` INTEGER NOT NULL, `responses` TEXT NOT NULL, `isHostBlockedByDnsServer` INTEGER NOT NULL)",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "askedServer",
"columnName": "askedServer",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "responseSource",
"columnName": "responseSource",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "questionTime",
"columnName": "questionTime",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "responseTime",
"columnName": "responseTime",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "responses",
"columnName": "responses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "isHostBlockedByDnsServer",
"columnName": "isHostBlockedByDnsServer",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": true
},
"indices": [],
"foreignKeys": []
},
{
"tableName": "DnsRule",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `stagingType` INTEGER NOT NULL, `type` INTEGER NOT NULL, `host` TEXT NOT NULL, `target` TEXT NOT NULL, `ipv6Target` TEXT, `importedFrom` INTEGER, `isWildcard` INTEGER NOT NULL, FOREIGN KEY(`importedFrom`) REFERENCES `HostSource`(`id`) ON UPDATE NO ACTION ON DELETE NO ACTION )",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "stagingType",
"columnName": "stagingType",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "type",
"columnName": "type",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "host",
"columnName": "host",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "target",
"columnName": "target",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "ipv6Target",
"columnName": "ipv6Target",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "importedFrom",
"columnName": "importedFrom",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "isWildcard",
"columnName": "isWildcard",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": true
},
"indices": [
{
"name": "index_DnsRule_importedFrom",
"unique": false,
"columnNames": [
"importedFrom"
],
"createSql": "CREATE INDEX IF NOT EXISTS `index_DnsRule_importedFrom` ON `${TABLE_NAME}` (`importedFrom`)"
},
{
"name": "index_DnsRule_host_type_stagingType",
"unique": true,
"columnNames": [
"host",
"type",
"stagingType"
],
"createSql": "CREATE UNIQUE INDEX IF NOT EXISTS `index_DnsRule_host_type_stagingType` ON `${TABLE_NAME}` (`host`, `type`, `stagingType`)"
}
],
"foreignKeys": [
{
"table": "HostSource",
"onDelete": "NO ACTION",
"onUpdate": "NO ACTION",
"columns": [
"importedFrom"
],
"referencedColumns": [
"id"
]
}
]
},
{
"tableName": "HostSource",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `enabled` INTEGER NOT NULL, `ruleCount` INTEGER, `checksum` TEXT, `name` TEXT NOT NULL, `source` TEXT NOT NULL, `whitelistSource` INTEGER NOT NULL)",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "enabled",
"columnName": "enabled",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "ruleCount",
"columnName": "ruleCount",
"affinity": "INTEGER",
"notNull": false
},
{
"fieldPath": "checksum",
"columnName": "checksum",
"affinity": "TEXT",
"notNull": false
},
{
"fieldPath": "name",
"columnName": "name",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "source",
"columnName": "source",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "whitelistSource",
"columnName": "whitelistSource",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"columnNames": [
"id"
],
"autoGenerate": true
},
"indices": [],
"foreignKeys": []
}
],
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '4b8a3541fc27086145381f661dbb9a39')"
]
}
}
\ No newline at end of file
......@@ -37,7 +37,7 @@ import com.frostnerd.smokescreen.database.repository.HostSourceRepository
@Database(entities = [CachedResponse::class, DnsQuery::class, DnsRule::class, HostSource::class], version = AppDatabase.currentVersion)
abstract class AppDatabase : RoomDatabase() {
companion object {
const val currentVersion:Int = 11
const val currentVersion:Int = 12
}
abstract fun cachedResponseDao(): CachedResponseDao
......
......@@ -121,11 +121,14 @@ val MIGRATION_10_11 = migration(10, 11) {
Logger.logIfOpen("DB_MIGRATION", "Migration from 10 to 11 completed")
}
val MIGRATION_11_12 = migration(11, 12) {
it.execSQL("ALTER TABLE `DnsQuery` ADD COLUMN `isHostBlockedByDnsServer` INTEGER NOT NULL DEFAULT 0")
}
private val INSTANCE by parameterizedLazy<AppDatabase, Context> {
Room.databaseBuilder(it, AppDatabase::class.java, "data")
.allowMainThreadQueries()
.addMigrations(MIGRATION_2_X, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10, MIGRATION_10_11)
.addMigrations(MIGRATION_2_X, MIGRATION_3_4, MIGRATION_4_5, MIGRATION_5_6, MIGRATION_6_7, MIGRATION_7_8, MIGRATION_8_9, MIGRATION_9_10, MIGRATION_10_11, MIGRATION_11_12)
.build()
}
......
......@@ -25,15 +25,17 @@ import com.google.gson.reflect.TypeToken
*/
class StringListConverter {
private val gson = Gson()
private val type = object:TypeToken<MutableList<String>>() {}.type
private val type = object:TypeToken<List<String>>() {}.type
@TypeConverter
fun stringToList(value: String): MutableList<String> {
return gson.fromJson(value, type)
fun stringToList(value: String): List<String> {
return if(value == "[]") emptyList()
else gson.fromJson(value, type)
}
@TypeConverter
fun someObjectListToString(value: MutableList<String>): String {
return gson.toJson(value)
fun someObjectListToString(value: List<String>): String {
return if(value.isEmpty()) "[]"
else gson.toJson(value)
}
}
\ No newline at end of file
......@@ -36,6 +36,21 @@ interface DnsQueryDao {
@Query("SELECT * FROM DnsQuery WHERE name LIKE '%' ||:hostPart || '%'")
fun getAllWithHostLive(hostPart:String): LiveData<List<DnsQuery>>
@Query("SELECT * FROM DnsQuery WHERE (responseSource='UPSTREAM' AND (:showForwarded=1 OR (:showBlockedByServer=1 AND responseTime>0))) " +
"OR (responseSource='CACHE' AND :showCache=1) " +
"OR (responseSource='LOCALRESOLVER' AND :showDnsRules=1) " +
"OR (responseSource='CACHE_AND_LOCALRESOLVER' AND (:showCache=1 OR :showDnsRules=1)) " +
"OR (responseSource='OTHER' AND :showForwarded=1)")
fun getAllWithFilterLive(showForwarded:Boolean, showCache:Boolean, showDnsRules:Boolean, showBlockedByServer:Boolean):LiveData<List<DnsQuery>>
@Query("SELECT * FROM DnsQuery WHERE (responseSource='UPSTREAM' AND (:showForwarded=1 OR (:showBlockedByServer=1 AND responseTime>0)) " +
"OR (responseSource='CACHE' AND :showCache=1) " +
"OR (responseSource='LOCALRESOLVER' AND :showDnsRules=1) " +
"OR (responseSource='CACHE_AND_LOCALRESOLVER' AND (:showCache=1 OR :showDnsRules=1)) " +
"OR (responseSource='OTHER' AND :showForwarded=1)) " +
"AND name LIKE '%' ||:hostPart || '%'")
fun getAllWithHostAndFilterLive(hostPart:String, showForwarded:Boolean, showCache:Boolean, showDnsRules:Boolean, showBlockedByServer:Boolean):LiveData<List<DnsQuery>>
@Query("SELECT MAX(id) FROM DnsQuery")
fun getLastInsertedId():Long
......
......@@ -10,6 +10,9 @@ import com.frostnerd.smokescreen.database.converters.DnsTypeConverter
import com.frostnerd.smokescreen.database.converters.QuerySourceConverter
import com.frostnerd.smokescreen.database.converters.StringListConverter
import com.frostnerd.smokescreen.database.recordFromBase64
import com.frostnerd.smokescreen.equalsAny
import org.minidns.record.A
import org.minidns.record.AAAA
import org.minidns.record.Record
/*
......@@ -40,13 +43,37 @@ data class DnsQuery(
var responseSource:QueryListener.Source,
val questionTime: Long,
var responseTime: Long = 0,
var responses: MutableList<String>
var responses: List<String>,
var isHostBlockedByDnsServer:Boolean = responsesBlockHost(responses)
) {
@delegate:Ignore
val shortName:String by lazy { calculateShortName() }
@delegate:Ignore
val parsedResponses:List<Record<*>> by lazy {
parseResponses(responses)
}
companion object {
private val SHORT_DOMAIN_REGEX = "^((?:[^.]{1,3}\\.)+)([\\w]{4,})(?:\\.(?:[^.]*))*\$".toRegex()
private fun responsesBlockHost(responses:List<String>):Boolean {
return parseResponses(responses).any {
(it.type == Record.TYPE.A && (it.payload as A).toString() == "0.0.0.0") ||
(it.type == Record.TYPE.AAAA && (it.payload as AAAA).toString().equalsAny("::0", "0:0:0:0:0:0:0:0"))
}
}
private fun parseResponses(responses:List<String>): List<Record<*>> {
if(responses.isEmpty()) return emptyList()
val parsedResponses = mutableListOf<Record<*>>()
for (response in responses) {
parsedResponses.add(recordFromBase64(response))
}
return parsedResponses
}
fun encodeResponse(record: Record<*>):String = Base64.encodeToString(record.toByteArray(), Base64.NO_WRAP)
}
/**
......@@ -63,16 +90,6 @@ data class DnsQuery(
} else return name
}
fun getParsedResponses(): List<Record<*>> {
val responses = mutableListOf<Record<*>>()
for (response in this.responses) {
responses.add(recordFromBase64(response))
}
return responses
}
fun encodeResponse(record: Record<*>):String = Base64.encodeToString(record.toByteArray(), Base64.NO_WRAP)
override fun toString(): String {
return "DnsQuery(id=$id, type=$type, name='$name', askedServer=$askedServer, questionTime=$questionTime, responseTime=$responseTime, responses={${responses.size}})"
}
......
package com.frostnerd.smokescreen.database.repository
import android.content.Context
import androidx.lifecycle.LiveData
import androidx.lifecycle.Transformations
import com.frostnerd.smokescreen.database.converters.StringListConverter
import com.frostnerd.smokescreen.database.dao.DnsQueryDao
import com.frostnerd.smokescreen.database.entities.DnsQuery
import com.frostnerd.smokescreen.dialog.QueryLogFilterDialog
import com.frostnerd.smokescreen.getPreferences
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
......@@ -34,6 +37,48 @@ import java.io.FileWriter
class DnsQueryRepository(private val dnsQueryDao: DnsQueryDao) {
fun getAllWithFilterLive(filterConfig: QueryLogFilterDialog.FilterConfig): LiveData<List<DnsQuery>> {
return dnsQueryDao.getAllWithFilterLive(filterConfig.showForwarded, filterConfig.showCache, filterConfig.showDnsrules, filterConfig.showBlockedByDns).let {
filterDnsQuery(filterConfig, it)
}
}
fun getAllWithHostAndFilterLive(hostPart:String, filterConfig: QueryLogFilterDialog.FilterConfig): LiveData<List<DnsQuery>> {
return dnsQueryDao.getAllWithHostAndFilterLive(hostPart, filterConfig.showForwarded, filterConfig.showCache, filterConfig.showDnsrules, filterConfig.showBlockedByDns).let {
filterDnsQuery(filterConfig, it)
}
}
private fun filterDnsQuery(filterConfig: QueryLogFilterDialog.FilterConfig, liveData: LiveData<List<DnsQuery>>):LiveData<List<DnsQuery>> {
if(filterConfig.showForwarded && filterConfig.showBlockedByDns) {
return liveData
} else{
val holdsAllData = liveData.value?.let {
dnsQueryDao.getCount() == it.size
} ?: false
return when {
holdsAllData -> {
liveData
}
filterConfig.showForwarded -> {
Transformations.map(liveData) {
it.filter {
!it.isHostBlockedByDnsServer
}
}
}
else -> { // showForwarded = false, showBlockedByDns = true
Transformations.map(liveData) {
it.filter {
it.isHostBlockedByDnsServer
}
}
}
}
}
}
fun insertAllAsync(dnsQueries:List<DnsQuery>): Job {
return GlobalScope.launch(Dispatchers.IO) {
dnsQueryDao.insertAll(dnsQueries)
......
package com.frostnerd.smokescreen.dialog
import android.content.Context
import androidx.appcompat.app.AlertDialog
import com.frostnerd.smokescreen.R
import com.frostnerd.smokescreen.getPreferences
import kotlinx.android.synthetic.main.dialog_querylog_filter.view.*
/*
* Copyright (C) 2020 Daniel Wolf (Ch4t4r)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* You can contact the developer at daniel.wolf@frostnerd.com.
*/
class QueryLogFilterDialog (
context: Context,
activeFilter:FilterConfig,
shortenDomains:Boolean,
onFilterConfigured:(filterConfig:FilterConfig, shortenDomainsInList:Boolean) -> Unit
): AlertDialog(context, context.getPreferences().theme.dialogStyle) {
init {
val view = layoutInflater.inflate(R.layout.dialog_querylog_filter, null, false)
setTitle(R.string.querylog_filter_title)
setView(view)
setButton(BUTTON_POSITIVE, context.getText(android.R.string.ok)) { _, _ ->
val filter = FilterConfig(
view.showForwarded.isChecked,
view.showCache.isChecked,
view.showDnsRules.isChecked,
view.showBlockedByDns.isChecked
)
onFilterConfigured(filter, view.shortenDomain.isChecked)
}
setButton(BUTTON_NEGATIVE, context.getString(android.R.string.cancel)) { _, _ -> }
view.showForwarded.isChecked = activeFilter.showForwarded
view.showCache.isChecked = activeFilter.showCache
view.showDnsRules.isChecked = activeFilter.showDnsrules
view.showBlockedByDns.isChecked = activeFilter.showBlockedByDns
if(shortenDomains) view.shortenDomain.isChecked = true
}
data class FilterConfig(
val showForwarded:Boolean,
val showCache:Boolean,
val showDnsrules:Boolean,
val showBlockedByDns:Boolean
) {
val showAll = showForwarded && showCache && showDnsrules && showBlockedByDns
companion object {
val showAllConfig = FilterConfig(true, true, true, true)
}
}
}
\ No newline at end of file
......@@ -92,7 +92,7 @@ class QueryLogDetailFragment : Fragment() {
createDnsRule.setOnClickListener {
val query = currentQuery
if(query != null) {
val answerIP = query.getParsedResponses().firstOrNull {
val answerIP = query.parsedResponses.firstOrNull {
it.type == query.type
}?.payload?.toString() ?: if(query.type == Record.TYPE.A) "0.0.0.0" else "::1"
DnsRuleDialog(requireContext(), DnsRule(query.type, query.name, target = answerIP), onRuleCreated = { newRule ->
......@@ -155,7 +155,7 @@ class QueryLogDetailFragment : Fragment() {
QueryListener.Source.LOCALRESOLVER -> getString(R.string.windows_querylogging_usedserver_dnsrules)
else -> query.askedServer?.replace("tls::", "")?.replace("https::", "") ?: "-"
}
responses.text = query.getParsedResponses().joinToString(separator = "\n") {
responses.text = query.parsedResponses.joinToString(separator = "\n") {
val payload = it.payload
"$payload (${it.type.name}, TTL: ${it.ttl})"
}.let {
......
......@@ -16,14 +16,12 @@ import com.frostnerd.lifecyclemanagement.launchWithLifecycle
import com.frostnerd.smokescreen.R
import com.frostnerd.smokescreen.database.entities.DnsQuery
import com.frostnerd.smokescreen.database.getDatabase
import com.frostnerd.smokescreen.equalsAny
import com.frostnerd.smokescreen.dialog.QueryLogFilterDialog
import com.frostnerd.smokescreen.fragment.QueryLogFragment
import com.frostnerd.smokescreen.util.LiveDataSource
import kotlinx.android.synthetic.main.fragment_querylog_list.*
import kotlinx.android.synthetic.main.fragment_querylog_list.view.*
import kotlinx.android.synthetic.main.item_logged_query.view.*
import org.minidns.record.A
import org.minidns.record.AAAA
import org.minidns.record.Record
/*
* Copyright (C) 2019 Daniel Wolf (Ch4t4r)
......@@ -46,6 +44,8 @@ import org.minidns.record.Record
class QueryLogListFragment: Fragment(), SearchView.OnQueryTextListener {