RuleImportService.kt 20.8 KB
Newer Older
1
2
package com.frostnerd.smokescreen.service

3
import android.app.IntentService
4
import android.app.NotificationManager
5
import android.app.PendingIntent
6
import android.content.Context
7
import android.content.Intent
Daniel Wolf's avatar
Daniel Wolf committed
8
import android.net.Uri
9
import android.os.IBinder
10
import androidx.core.app.NotificationCompat
11
import com.frostnerd.general.service.isServiceRunning
12
import com.frostnerd.smokescreen.*
13
14
15
import com.frostnerd.smokescreen.database.entities.DnsRule
import com.frostnerd.smokescreen.database.entities.HostSource
import com.frostnerd.smokescreen.database.getDatabase
16
import com.frostnerd.smokescreen.util.DeepActionState
17
import com.frostnerd.smokescreen.util.LanguageContextWrapper
18
import com.frostnerd.smokescreen.util.Notifications
19
import com.frostnerd.smokescreen.util.RequestCodes
20
21
import okhttp3.OkHttpClient
import okhttp3.Request
22
import okhttp3.Response
23
import org.minidns.record.Record
Daniel Wolf's avatar
Daniel Wolf committed
24
25
26
import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
import java.util.regex.Matcher
import java.util.regex.Pattern

/*
 * Copyright (C) 2019 Daniel Wolf (Ch4t4r)
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.

 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.

 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * You can contact the developer at daniel.wolf@frostnerd.com.
 */
Daniel Wolf's avatar
Daniel Wolf committed
48
@Suppress("DEPRECATION")
49
class RuleImportService : IntentService("RuleImportService") {
Daniel Wolf's avatar
Daniel Wolf committed
50
    private val dnsmasqMatcher =
51
52
        Pattern.compile("^address=/([^/]+)/(?:([0-9.]+)|([0-9a-fA-F:]+))(?:/?\$|\\s+.*)").matcher("") // address=/xyz.com/0.0.0.0
    private val dnsmasqBlockMatcher = Pattern.compile("^address=/([^/]+)/$").matcher("") // address=/xyz.com/
Daniel Wolf's avatar
Daniel Wolf committed
53
    private val hostsMatcher =
Daniel Wolf's avatar
Daniel Wolf committed
54
        Pattern.compile("^((?:[A-Fa-f0-9:.])+)\\s+([^\\s]+)")
55
56
57
            .matcher("") // 0.0.0.0 xyz.com
    private val domainsMatcher = Pattern.compile("^([_\\w*][*\\w_\\-.]+)(?:\$|\\s+.*)").matcher("") // xyz.com
    private val adblockMatcher = Pattern.compile("^\\|\\|(.*)\\^(?:\$|\\s+.*)").matcher("") // ||xyz.com^
58
    private val ruleCommitSize = 10000
59
    private var notification: NotificationCompat.Builder? = null
60
    private var ruleCount: Int = 0
Daniel Wolf's avatar
Daniel Wolf committed
61
    private var newlyAddedRuleCount:Int = 0
62
    private var isAborted = false
63
    private lateinit var sources:List<HostSource>
64
    private lateinit var sourcesIds:List<Long>
65

66
67
68
69
    companion object {
        const val BROADCAST_IMPORT_DONE = "com.frostnerd.nebulo.RULE_IMPORT_DONE"
    }

70
    private val httpClient by lazy(LazyThreadSafetyMode.NONE) {
71
72
73
        OkHttpClient()
    }

74
75
76
77
    override fun attachBaseContext(newBase: Context) {
        super.attachBaseContext(LanguageContextWrapper.attachFromSettings(this, newBase))
    }

Daniel Wolf's avatar
Daniel Wolf committed
78
79
    override fun onCreate() {
        super.onCreate()
80
        (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager).cancel(Notifications.ID_DNSRULE_IMPORT_FINISHED)
Daniel Wolf's avatar
Daniel Wolf committed
81
82
    }

83
    override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
84
        return if (intent != null && intent.hasExtra("abort")) {
85
            abortImport()
86
87
88
89
            START_NOT_STICKY
        } else super.onStartCommand(intent, flags, startId)
    }

Daniel Wolf's avatar
Daniel Wolf committed
90
    override fun onHandleIntent(intent: Intent?) {
91
        createNotification()
92
93
94
95
        sources = (intent?.takeIf { it.hasExtra("sources") }?.getLongArrayExtra("sources")?.let { ids ->
            getDatabase().hostSourceDao().getAllEnabled().filter {
                it.id in ids
            }
96
        } ?: getDatabase().hostSourceDao().getAllEnabled()).sortedWith(compareBy({it.whitelistSource}, {it.name}))
97
        sourcesIds = sources.map { it.id }
98
        startWork()
99
100
    }

101
    private fun abortImport() {
102
        isAborted = true
103
104
    }

105
106
    private fun createNotification() {
        if (notification == null) {
107
            notification = NotificationCompat.Builder(this, Notifications.getDnsRuleChannelId(this))
108
109
110
111
112
113
114
115
            notification!!.setSmallIcon(R.drawable.ic_mainnotification)
            notification!!.setOngoing(true)
            notification!!.setAutoCancel(false)
            notification!!.setSound(null)
            notification!!.setOnlyAlertOnce(true)
            notification!!.setUsesChronometer(true)
            notification!!.setContentTitle(getString(R.string.notification_ruleimport_title))
            notification!!.setContentText(getString(R.string.notification_ruleimport_secondarymessage))
116
            notification!!.setStyle(NotificationCompat.BigTextStyle().bigText(getString(R.string.notification_ruleimport_secondarymessage)))
117
            notification!!.setProgress(100, 0, true)
118
            notification!!.setContentIntent(DeepActionState.DNS_RULES.pendingIntentTo(this))
119
120
            val abortPendingAction = PendingIntent.getService(
                this,
121
                RequestCodes.RULE_IMPORT_ABORT,
122
123
124
125
                Intent(this, RuleImportService::class.java).putExtra("abort", true),
                PendingIntent.FLAG_CANCEL_CURRENT
            )
            val abortAction =
126
                NotificationCompat.Action(R.drawable.ic_times, getString(android.R.string.cancel), abortPendingAction)
127
            notification!!.addAction(abortAction)
128
        }
Daniel Wolf's avatar
Daniel Wolf committed
129
        startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
130
131
    }

132
    private fun showSuccessNotification() {
133
        val successNotification = NotificationCompat.Builder(this, Notifications.getDnsRuleChannelId(this))
134
135
136
        successNotification.setSmallIcon(R.drawable.ic_mainnotification)
        successNotification.setAutoCancel(true)
        successNotification.setContentTitle(getString(R.string.notification_ruleimportfinished_title))
137
        getString(R.string.notification_ruleimportfinished_message,
138
139
            getPreferences().numberFormatter.format(ruleCount),
            getPreferences().numberFormatter.format(ruleCount - newlyAddedRuleCount)).apply {
140
141
142
            successNotification.setContentText(this)
            successNotification.setStyle(NotificationCompat.BigTextStyle().bigText(this))
        }
143
        successNotification.setContentIntent(DeepActionState.DNS_RULES.pendingIntentTo(this))
Daniel Wolf's avatar
Daniel Wolf committed
144
        (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager).notify(Notifications.ID_DNSRULE_IMPORT_FINISHED, successNotification.build())
145
146
    }

147
    private fun updateNotification(source: HostSource, count: Int, maxCount: Int) {
148
        if (notification != null) {
149
150
151
152
153
            val notificationText = getString(
                R.string.notification_ruleimport_message,
                source.name,
                source.source
            )
154
            notification?.setContentText(
155
                notificationText
156
            )
157
            notification?.setStyle(NotificationCompat.BigTextStyle().bigText(notificationText))
158
            notification?.setProgress(maxCount, count, false)
Daniel Wolf's avatar
Daniel Wolf committed
159
            startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
160
161
162
        }
    }

163
    private fun updateNotificationFinishing() {
164
        if (notification != null) {
165
166
167
168
169
170
            val notificationText = getString(R.string.notification_ruleimport_tertiarymessage)
            notification?.setContentText(
                notificationText
            )
            notification?.setStyle(NotificationCompat.BigTextStyle().bigText(notificationText))
            notification?.setProgress(1, 1, true)
Daniel Wolf's avatar
Daniel Wolf committed
171
            startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
172
173
174
        }
    }

175
    private fun startWork() {
176
        val dnsRuleDao = getDatabase().dnsRuleDao()
177
178
179
180
181
182
        if(sourcesIds.isEmpty()) {
            dnsRuleDao.deleteAllNonUserRules()
        } else {
            dnsRuleDao.markNonUserRulesForDeletion(sourcesIds) // Stage all, ignoring if the source is actually processed in this run
            dnsRuleDao.deleteStagedRules()
        }
183
        var count = 0
184
        val newChecksums = mutableMapOf<HostSource, String>()
185
        sources.forEach {
186
187
            log("Importing HostSource $it")
            if (!isAborted) {
188
                updateNotification(it, count, sources.size)
189
190
191
192
193
194
195
                count++
                if (it.isFileSource) {
                    log("Importing from file")
                    var stream: InputStream? = null
                    try {
                        val uri = Uri.parse(it.source)
                        stream = contentResolver.openInputStream(uri)
Daniel Wolf's avatar
Daniel Wolf committed
196
                        processLines(it, stream!!)
197
198
199
200
201
                    } catch (ex: Exception) {
                        log("Import failed: $ex")
                        ex.printStackTrace()
                    } finally {
                        stream?.close()
202
                    }
Daniel Wolf's avatar
Daniel Wolf committed
203
                } else {
204
205
206
207
                    log("Importing from URL")
                    var response: Response? = null
                    try {
                        val request = Request.Builder().url(it.source)
208
209
                        val realChecksum = it.checksum?.replace("<qt>", "\"")
                        if(realChecksum != null) request.header("If-None-Match", realChecksum)
210
                        response = httpClient.newCall(request.build()).execute()
Daniel Wolf's avatar
Daniel Wolf committed
211
212
                        val receivedChecksum = response.headers.find { header ->
                            header.first.equals("etag", true)
213
                        }?.second
214
                        val localDataIsRecent = response.code == 304 || (realChecksum != null && (receivedChecksum == realChecksum || receivedChecksum == "W/$realChecksum"))
215
216
                        when {
                            response.isSuccessful && !localDataIsRecent -> {
Daniel Wolf's avatar
Daniel Wolf committed
217
218
                                response.headers.find { header ->
                                    header.first.equals("etag", true)
219
                                }?.second?.apply {
220
                                    newChecksums[it] = this.replace("\"", "<qt>")
221
                                }
Daniel Wolf's avatar
Daniel Wolf committed
222
                                log("Now parsing content...")
223
224
                                processLines(it, response.body!!.byteStream())
                            }
225
                            localDataIsRecent -> {
226
227
228
229
230
                                log("Host source ${it.name} hasn't changed, not updating.")
                                ruleCount += it.ruleCount ?: 0
                                dnsRuleDao.unstageRulesOfSource(it.id)
                                log("Unstaged rules for ${it.name}")
                            }
231
                            else -> log("Downloading resource of ${it.name} failed (response=${response.code}.")
232
233
                        }
                    } catch (ex: java.lang.Exception) {
234
                        ex.printStackTrace()
235
                        log("Downloading resource of ${it.name} failed ($ex)")
236
                        newChecksums.remove(it)
237
238
239
                    } finally {
                        response?.body?.close()
                    }
Daniel Wolf's avatar
Daniel Wolf committed
240
                }
241
242
243
244
                log("Import of $it finished")
            } else {
                log("Aborting import.")
                return@forEach
245
            }
246
247
248
        }
        if (!isAborted) {
            updateNotificationFinishing()
249
250
251
252
253
254
255
            if(sourcesIds.isNotEmpty()) {
                log("Delete rules staged for deletion")
                dnsRuleDao.deleteMarkedRules()
                log("Committing staging")
                dnsRuleDao.commitStaging()
                dnsRuleDao.deleteStagedRules()
            }
256
257
            log("Recreating database indices")
            getDatabase().recreateDnsRuleIndizes()
258
259
260
261
262
263
            log("Updating Etag values for sources")
            newChecksums.forEach { (source, etag) ->
                source.checksum = etag
                getDatabase().hostSourceDao().update(source)
            }
            getDatabase().hostSourceDao().removeChecksumForDisabled()
264
            log("Done.")
265
            if(isServiceRunning(DnsVpnService::class.java)) DnsVpnService.restartVpn(this, false)
Daniel Wolf's avatar
Daniel Wolf committed
266
            newlyAddedRuleCount = sourcesIds.sumBy { dnsRuleDao.getCountForHostSource(it) }
267
268
            showSuccessNotification()
        } else {
269
270
271
            dnsRuleDao.deleteStagedRules()
            dnsRuleDao.commitStaging()
            sendLocalBroadcast(Intent(BROADCAST_IMPORT_DONE))
272
            stopForeground(true)
273
        }
274
275
276
        log("All imports finished.")
        stopForeground(true)
        sendLocalBroadcast(Intent(BROADCAST_IMPORT_DONE))
277
278
    }

279
    private fun processLines(source: HostSource, stream: InputStream) {
280
        val parsers = mutableMapOf(
Daniel Wolf's avatar
Daniel Wolf committed
281
282
283
284
285
            dnsmasqMatcher to (0 to mutableListOf<DnsRule>()),
            hostsMatcher to (0 to mutableListOf()),
            domainsMatcher to (0 to mutableListOf()),
            adblockMatcher to (0 to mutableListOf()),
            dnsmasqBlockMatcher to (0 to mutableListOf())
286
        )
287
        var lineCount = 0
288
        var ruleCount = 0
289
        BufferedReader(InputStreamReader(stream)).useLines { lines ->
290
291
            var remainingMatcher:Matcher? = null
            var hostsOfRemainingMatcher:MutableList<DnsRule>? = null
292
            var successfulMatches = 0
293
294
            lines.forEach { _line ->
                val line = _line.trim()
295
                if (!isAborted) {
Daniel Wolf's avatar
Daniel Wolf committed
296
                    if (parsers.isNotEmpty() && !line.startsWith("#") && !line.startsWith("!") && line.isNotBlank()) {
297
                        lineCount++
298
299
                        if(remainingMatcher != null) {
                            if (remainingMatcher!!.reset(line).matches()) {
300
                                val rule = processLine(remainingMatcher!!, source, source.whitelistSource)
301
302
                                hostsOfRemainingMatcher!!.addAll(rule.onEach {
                                    it.stagingType = 2
Daniel Wolf's avatar
Daniel Wolf committed
303
                                })
304
                                if (lineCount > ruleCommitSize) {
305
                                    ruleCount += commitLines(parsers)
306
307
                                    lineCount = 0
                                }
308
309
310
311
312
                            }
                        } else {
                            val iterator = parsers.iterator()
                            for ((matcher, hosts) in iterator) {
                                if (matcher.reset(line).matches()) {
313
                                    successfulMatches+=1
314
                                    val rule = processLine(matcher, source, source.whitelistSource)
315
316
                                    hosts.second.addAll(rule.onEach {
                                        it.stagingType = 2
317
318
319
320
321
322
                                    })
                                    if (lineCount > ruleCommitSize) {
                                        ruleCount += commitLines(parsers)
                                        lineCount = 0
                                    }
                                } else {
323
                                    if(successfulMatches > 35) {
324
325
                                        remainingMatcher = parsers.keys.first()
                                        hostsOfRemainingMatcher = parsers.values.first().second
326
327
328
329
330
331
332
333
334
                                    } else {
                                        if (hosts.first > 5) {
                                            log("Matcher $matcher failed 5 times, last for '$line'. Removing.")
                                            iterator.remove()
                                        } else parsers[matcher] = hosts.copy(hosts.first + 1)
                                        if (parsers.isEmpty()) {
                                            log("No parsers left. Aborting.")
                                            return@forEach
                                        }
335
                                    }
336
                                }
337
                            }
338
339
                        }
                    }
340
341
                } else {
                    return@useLines
342
343
344
                }
            }
        }
345
        if(!isAborted) {
346
347
348
            if (parsers.isNotEmpty()) {
                ruleCount += commitLines(parsers, true)
            }
349
350
351
            source.ruleCount = ruleCount
            getDatabase().hostSourceDao().update(source)
        }
352
353
    }

354
    private fun commitLines(
355
        parsers: Map<Matcher, Pair<Int, MutableList<DnsRule>>>,
356
        forceCommit: Boolean = false
357
    ):Int {
358
        val hosts = parsers[parsers.keys.minByOrNull {
Daniel Wolf's avatar
Daniel Wolf committed
359
360
            parsers[it]?.first!!
        } ?: parsers.keys.first()]?.second!!
361
        return if (hosts.size > ruleCommitSize || forceCommit) {
362
            getDatabase().dnsRuleDao().insertAllIgnoreConflict(hosts)
363
            ruleCount += hosts.size
364
            val added = hosts.size
365
            hosts.clear()
366
367
            added
        } else 0
368
369
    }

370
    private val wwwRegex = Regex("^www\\.")
Daniel Wolf's avatar
Daniel Wolf committed
371
    private fun processLine(matcher: Matcher, source: HostSource, isWhitelist:Boolean): Array<DnsRule> {
372
373
        val defaultTargetV4 = if(isWhitelist) "" else "0"
        val defaultTargetV6 = if(isWhitelist) "" else "1"
374
        when {
375
            matcher.groupCount() == 1 -> {
Daniel Wolf's avatar
Daniel Wolf committed
376
                val host = if(matcher == dnsmasqBlockMatcher) "%%" + matcher.group(1)
Daniel Wolf's avatar
Daniel Wolf committed
377
                else matcher.group(1)!!.replace(wwwRegex, "")
378
                return createRule(host, defaultTargetV4, defaultTargetV6, Record.TYPE.ANY, source)
379
            }
Daniel Wolf's avatar
Daniel Wolf committed
380
            matcher == dnsmasqMatcher -> {
Daniel Wolf's avatar
Daniel Wolf committed
381
                val host = "%%" + matcher.group(1)
Daniel Wolf's avatar
Daniel Wolf committed
382
                var target = matcher.group(2)!!
383
384
                val type = if (target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A
                target = target.let {
385
                    when (it) {
386
                        "0.0.0.0", "::", "::0" -> "0"
387
                        "127.0.0.1", "::1" -> "1"
388
389
                        else -> it
                    }
390
                }
391
                return createRule(host, target, null, type, source)
392
            }
Daniel Wolf's avatar
Daniel Wolf committed
393
            matcher == hostsMatcher -> {
394
                return if(isWhitelist) {
Daniel Wolf's avatar
Daniel Wolf committed
395
                    val host = matcher.group(2)!!.replace(wwwRegex, "")
Daniel Wolf's avatar
Daniel Wolf committed
396
                    arrayOf(
397
398
399
400
401
402
403
404
                        DnsRule(
                            Record.TYPE.ANY,
                            host,
                            defaultTargetV4,
                            defaultTargetV6,
                            importedFrom = source.id
                        )
                    )
405
                } else {
Daniel Wolf's avatar
Daniel Wolf committed
406
                    var target = matcher.group(1)!!
407
408
409
                    val type = if (target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A
                    target = target.let {
                        when (it) {
410
                            "0.0.0.0", "::", "::0" -> "0"
411
412
413
                            "127.0.0.1", "::1" -> "1"
                            else -> it
                        }
414
                    }
Daniel Wolf's avatar
Daniel Wolf committed
415
                    val host = matcher.group(2)!!.replace(wwwRegex, "")
416
                    return createRule(host, target, null, type, source)
417
                }
418
419
420
421
422
            }
        }
        throw IllegalStateException()
    }

Daniel Wolf's avatar
Daniel Wolf committed
423
    private val wildcardNormalisationRegex = Regex("(?:^\\*\\*\\.)|(\\.\\*\\*$)")
424
    private val leadingWildcardRegex = Regex("^\\*\\.")
425
426
427
428
429
430
    private fun createRule(
        host: String,
        target: String,
        targetV6: String? = null,
        type: Record.TYPE,
        source: HostSource
Daniel Wolf's avatar
Daniel Wolf committed
431
    ): Array<DnsRule> {
432
        var isWildcard = false
Daniel Wolf's avatar
Daniel Wolf committed
433
        val alteredHost:Array<String> = host.let {
434
            if (it.contains("*")) {
435
                isWildcard = true
436
                if (source.isFileSource) {
Daniel Wolf's avatar
Daniel Wolf committed
437
                    arrayOf(
438
439
440
                        it.replace("**", "%%").replace("*", "%")
                            .replace(wildcardNormalisationRegex, "**")
                    )
441
                } else {
442
                    if (it.startsWith("*.")) {
Daniel Wolf's avatar
Daniel Wolf committed
443
                        arrayOf(
444
445
446
447
448
449
                            it.replace(leadingWildcardRegex, ""),
                            it.replace(leadingWildcardRegex, "**.").replace("**", "%%")
                                .replace("*", "%%")
                                .replace(wildcardNormalisationRegex, "**")
                        )
                    } else {
Daniel Wolf's avatar
Daniel Wolf committed
450
                        arrayOf(
451
452
453
454
455
                            it.replace("**", "%%")
                                .replace("*", "%%")
                                .replace(wildcardNormalisationRegex, "**")
                        )
                    }
456
                }
Daniel Wolf's avatar
Daniel Wolf committed
457
            } else arrayOf(it)
458
        }
459
460
        return alteredHost.map {
            DnsRule(type, it, target, targetV6, source.id, isWildcard)
Daniel Wolf's avatar
Daniel Wolf committed
461
        }.toTypedArray()
462
463
    }

464
465
    override fun onDestroy() {
        super.onDestroy()
466
        abortImport()
467
468
469
470
471
    }

    override fun onBind(intent: Intent?): IBinder? {
        return null
    }
472
}