Commit 44443e42 authored by Daniel Wolf's avatar Daniel Wolf
Browse files

Added service which imports hosts from http sources

parent 71d4d9a4
Pipeline #4798 passed with stage
in 52 seconds
......@@ -2,7 +2,7 @@
"formatVersion": 1,
"database": {
"version": 6,
"identityHash": "be58818eae851a6b386c96630cd3b3c2",
"identityHash": "4ca420a8d5d5420e65d499a53e6f16b1",
"entities": [
{
"tableName": "CachedResponse",
......@@ -101,7 +101,7 @@
},
{
"tableName": "DnsRule",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `type` INTEGER NOT NULL, `host` TEXT NOT NULL, `ttl` INTEGER NOT NULL, `record` TEXT NOT NULL, `importedFrom` INTEGER, FOREIGN KEY(`importedFrom`) REFERENCES `HostSource`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `type` INTEGER NOT NULL, `host` TEXT NOT NULL, `target` TEXT NOT NULL, `importedFrom` INTEGER, FOREIGN KEY(`importedFrom`) REFERENCES `HostSource`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "id",
......@@ -122,14 +122,8 @@
"notNull": true
},
{
"fieldPath": "ttl",
"columnName": "ttl",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "record",
"columnName": "record",
"fieldPath": "target",
"columnName": "target",
"affinity": "TEXT",
"notNull": true
},
......@@ -164,13 +158,12 @@
"createSql": "CREATE INDEX `index_DnsRule_host` ON `${TABLE_NAME}` (`host`)"
},
{
"name": "index_DnsRule_host_type",
"name": "index_DnsRule_host_isIpv6",
"unique": false,
"columnNames": [
"host",
"type"
"host"
],
"createSql": "CREATE INDEX `index_DnsRule_host_type` ON `${TABLE_NAME}` (`host`, `type`)"
"createSql": "CREATE INDEX `index_DnsRule_host_isIpv6` ON `${TABLE_NAME}` (`host`)"
}
],
"foreignKeys": [
......@@ -229,7 +222,7 @@
"views": [],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'be58818eae851a6b386c96630cd3b3c2')"
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '4ca420a8d5d5420e65d499a53e6f16b1')"
]
}
}
\ No newline at end of file
......@@ -202,6 +202,7 @@
<action android:name="android.service.quicksettings.action.QS_TILE"/>
</intent-filter>
</service>
<service android:name=".service.RuleImportService"/>
<receiver
android:name=".receiver.AutostartReceiver"
......
package com.frostnerd.smokescreen.activity
import android.content.Intent
import android.os.Bundle
import android.view.View
import androidx.recyclerview.widget.LinearLayoutManager
......@@ -12,6 +13,7 @@ import com.frostnerd.smokescreen.R
import com.frostnerd.smokescreen.database.entities.HostSource
import com.frostnerd.smokescreen.database.getDatabase
import com.frostnerd.smokescreen.getPreferences
import com.frostnerd.smokescreen.service.RuleImportService
import com.frostnerd.smokescreen.util.SpaceItemDecorator
import kotlinx.android.synthetic.main.activity_dns_rules.*
import kotlinx.android.synthetic.main.activity_dns_rules.toolBar
......@@ -62,6 +64,9 @@ class DnsRuleActivity : BaseActivity() {
getDatabase().hostSourceDao().insert(newSource)
}
}
refresh.setOnClickListener {
startService(Intent(this, RuleImportService::class.java))
}
sourceAdapterList = getDatabase().hostSourceDao().getAll().toMutableList()
adapterDataSource = ListDataSource(sourceAdapterList)
sourceAdapter = ModelAdapterBuilder.withModelAndViewHolder({ view, type ->
......
......@@ -51,8 +51,11 @@ private val MIGRATION_4_5 = migration(4, 5) {
}
private val MIGRATION_5_6 = migration(5, 6) {
Logger.logIfOpen("DB_MIGRATION", "Migrating from 5 to 6")
it.execSQL("CREATE TABLE IF NOT EXISTS `DnsRule` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `type` INTEGER NOT NULL, `host` TEXT NOT NULL, `ttl` INTEGER NOT NULL, `record` TEXT NOT NULL, `importedFrom` INTEGER, FOREIGN KEY(`importedFrom`) REFERENCES `HostSource`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )")
it.execSQL("CREATE TABLE IF NOT EXISTS `DnsRule` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `type` INTEGER NOT NULL, `host` TEXT NOT NULL, `target` TEXT NOT NULL, `importedFrom` INTEGER, FOREIGN KEY(`importedFrom`) REFERENCES `HostSource`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )")
it.execSQL("CREATE TABLE IF NOT EXISTS `HostSource` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `name` TEXT NOT NULL, `source` TEXT NOT NULL, `enabled` INTEGER NOT NULL)")
it.execSQL("CREATE INDEX `index_DnsRule_importedFrom` ON `DnsRule` (`importedFrom`)")
it.execSQL("CREATE INDEX `index_DnsRule_host` ON `DnsRule` (`host`)")
it.execSQL("CREATE INDEX `index_DnsRule_host_type` ON `DnsRule` (`host`, `type`)")
Logger.logIfOpen("DB_MIGRATION", "Migration from 5 to 6 completed")
}
......
package com.frostnerd.smokescreen.database.dao
import androidx.room.Dao
import androidx.room.Insert
import androidx.room.Query
import com.frostnerd.smokescreen.database.entities.DnsRule
/*
* Copyright (C) 2019 Daniel Wolf (Ch4t4r)
......@@ -29,4 +31,10 @@ interface DnsRuleDao {
@Query("DELETE FROM DnsRule WHERE importedFrom IS NULL")
fun deleteAllExceptUserRules()
@Insert
fun insertAll(rules:Collection<DnsRule>)
@Query("SELECT COUNT(*) FROM DnsRule")
fun getCount():Long
}
\ No newline at end of file
......@@ -34,10 +34,9 @@ import org.minidns.record.Record
)
@TypeConverters(DnsTypeConverter::class)
data class DnsRule(
val type: Record.TYPE,
val type:Record.TYPE,
val host: String,
val ttl: Long,
val record: String,
val target: String,
val importedFrom: Long? = null
) {
@PrimaryKey(autoGenerate = true) var id: Long = 0
......
package com.frostnerd.smokescreen.service
import android.app.Service
import android.content.Intent
import android.os.IBinder
import com.frostnerd.smokescreen.database.entities.DnsRule
import com.frostnerd.smokescreen.database.entities.HostSource
import com.frostnerd.smokescreen.database.getDatabase
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import okhttp3.OkHttpClient
import okhttp3.Request
import org.minidns.record.Record
import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
import java.lang.IllegalStateException
import java.util.regex.Matcher
import java.util.regex.Pattern
/*
* Copyright (C) 2019 Daniel Wolf (Ch4t4r)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* You can contact the developer at daniel.wolf@frostnerd.com.
*/
class RuleImportService : Service() {
private var importJob: Job? = null
private val DNSMASQ_MATCHER = Pattern.compile("^address=/([^/]+)/(?:([0-9.]+)|([0-9a-fA-F:]+))").matcher("")
private val HOSTS_MATCHER =
Pattern.compile("^((?:[A-Fa-f0-9:]|[0-9.])+)\\s+([a-zA-Z0-9.]+).*")
.matcher("")
private val DOMAINS_MATCHER = Pattern.compile("^([A-Za-z0-9][A-Za-z0-9\\-.]+)").matcher("")
private val ADBLOCK_MATCHER = Pattern.compile("^\\|\\|(.*)\\^$").matcher("")
private val httpClient by lazy {
OkHttpClient()
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
startWork()
return START_STICKY
}
private fun startWork() {
importJob = GlobalScope.launch {
getDatabase().dnsRuleDao().deleteAllExceptUserRules()
getDatabase().hostSourceDao().getAll().forEach {
if (it.isFileSource) {
TODO()
} else {
val request = Request.Builder().url(it.source)
val response = httpClient.newCall(request.build()).execute()
if (response.isSuccessful) {
processLines(it, response.body()!!.byteStream())
}
}
}
importJob = null
stopSelf()
}
}
private fun processLines(source:HostSource, stream: InputStream) {
val parsers = mutableMapOf(
DNSMASQ_MATCHER to mutableListOf<Host>(),
HOSTS_MATCHER to mutableListOf(),
DOMAINS_MATCHER to mutableListOf(),
ADBLOCK_MATCHER to mutableListOf()
)
BufferedReader(InputStreamReader(stream)).useLines { lines ->
lines.forEach { line ->
if (!line.trim().startsWith("#") && !line.trim().startsWith("!") && !line.isBlank()) {
val iterator = parsers.iterator()
for ((matcher, hosts) in iterator) {
if (matcher.reset(line).matches()) {
hosts.add(processLine(matcher))
commitLines(source, parsers)
} else {
iterator.remove()
}
}
}
}
}
commitLines(source, parsers, true)
}
private fun commitLines(source:HostSource, parsers:Map<Matcher, MutableList<Host>>, forceCommit:Boolean = false) {
if(parsers.size == 1) {
val hosts = parsers[parsers.keys.first()]!!
if(hosts.size > 5000 || forceCommit) {
getDatabase().dnsRuleDao().insertAll(hosts.map {
DnsRule(it.type, it.host, it.target, source.id)
})
hosts.clear()
}
}
}
private fun processLine(matcher:Matcher):Host {
when {
matcher.groupCount() == 1 -> return Host(matcher.group(1), "0.0.0.0", Record.TYPE.ANY)
matcher == DNSMASQ_MATCHER -> {
val host = matcher.group(1)
val target = matcher.group(2)
return Host(host, target, if(target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A)
}
matcher == HOSTS_MATCHER -> {
val target = matcher.group(1)
val host = matcher.group(2)
return Host(host, target, if(target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A)
}
}
throw IllegalStateException()
}
override fun onDestroy() {
super.onDestroy()
importJob?.cancel()
}
override fun onBind(intent: Intent?): IBinder? {
return null
}
private data class Host(val host: String, val target:String, val type: Record.TYPE)
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment