RuleImportService.kt 19.8 KB
Newer Older
1
2
package com.frostnerd.smokescreen.service

3
import android.app.IntentService
4
import android.app.NotificationManager
5
import android.app.PendingIntent
6
import android.content.Context
7
import android.content.Intent
Daniel Wolf's avatar
Daniel Wolf committed
8
import android.net.Uri
9
import android.os.IBinder
10
import androidx.core.app.NotificationCompat
11
import com.frostnerd.general.service.isServiceRunning
12
import com.frostnerd.smokescreen.*
13
14
15
import com.frostnerd.smokescreen.database.entities.DnsRule
import com.frostnerd.smokescreen.database.entities.HostSource
import com.frostnerd.smokescreen.database.getDatabase
16
import com.frostnerd.smokescreen.util.DeepActionState
17
import com.frostnerd.smokescreen.util.LanguageContextWrapper
18
import com.frostnerd.smokescreen.util.Notifications
19
import com.frostnerd.smokescreen.util.RequestCodes
20
21
import okhttp3.OkHttpClient
import okhttp3.Request
22
import okhttp3.Response
23
import org.minidns.record.Record
Daniel Wolf's avatar
Daniel Wolf committed
24
25
26
import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
import java.util.regex.Matcher
import java.util.regex.Pattern

/*
 * Copyright (C) 2019 Daniel Wolf (Ch4t4r)
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.

 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.

 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * You can contact the developer at daniel.wolf@frostnerd.com.
 */
Daniel Wolf's avatar
Daniel Wolf committed
48
@Suppress("DEPRECATION")
49
class RuleImportService : IntentService("RuleImportService") {
Daniel Wolf's avatar
Daniel Wolf committed
50
    private val dnsmasqMatcher =
51
52
        Pattern.compile("^address=/([^/]+)/(?:([0-9.]+)|([0-9a-fA-F:]+))(?:/?\$|\\s+.*)").matcher("") // address=/xyz.com/0.0.0.0
    private val dnsmasqBlockMatcher = Pattern.compile("^address=/([^/]+)/$").matcher("") // address=/xyz.com/
Daniel Wolf's avatar
Daniel Wolf committed
53
    private val hostsMatcher =
Daniel Wolf's avatar
Daniel Wolf committed
54
        Pattern.compile("^((?:[A-Fa-f0-9:.])+)\\s+([^\\s]+)")
55
56
57
            .matcher("") // 0.0.0.0 xyz.com
    private val domainsMatcher = Pattern.compile("^([_\\w*][*\\w_\\-.]+)(?:\$|\\s+.*)").matcher("") // xyz.com
    private val adblockMatcher = Pattern.compile("^\\|\\|(.*)\\^(?:\$|\\s+.*)").matcher("") // ||xyz.com^
58
    private val ruleCommitSize = 10000
59
    private var notification: NotificationCompat.Builder? = null
60
    private var ruleCount: Int = 0
Daniel Wolf's avatar
Daniel Wolf committed
61
    private var newlyAddedRuleCount:Int = 0
62
    private var isAborted = false
63
    private lateinit var sources:List<HostSource>
64
    private lateinit var sourcesIds:List<Long>
65

66
67
68
69
    companion object {
        const val BROADCAST_IMPORT_DONE = "com.frostnerd.nebulo.RULE_IMPORT_DONE"
    }

70
    private val httpClient by lazy(LazyThreadSafetyMode.NONE) {
71
72
73
        OkHttpClient()
    }

74
75
76
77
    override fun attachBaseContext(newBase: Context) {
        super.attachBaseContext(LanguageContextWrapper.attachFromSettings(this, newBase))
    }

Daniel Wolf's avatar
Daniel Wolf committed
78
79
    override fun onCreate() {
        super.onCreate()
80
        (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager).cancel(Notifications.ID_DNSRULE_IMPORT_FINISHED)
Daniel Wolf's avatar
Daniel Wolf committed
81
82
    }

83
    override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
84
        return if (intent != null && intent.hasExtra("abort")) {
85
            abortImport()
86
87
88
89
            START_NOT_STICKY
        } else super.onStartCommand(intent, flags, startId)
    }

Daniel Wolf's avatar
Daniel Wolf committed
90
    override fun onHandleIntent(intent: Intent?) {
91
        createNotification()
92
93
94
95
        sources = (intent?.takeIf { it.hasExtra("sources") }?.getLongArrayExtra("sources")?.let { ids ->
            getDatabase().hostSourceDao().getAllEnabled().filter {
                it.id in ids
            }
96
        } ?: getDatabase().hostSourceDao().getAllEnabled()).sortedWith(compareBy({it.whitelistSource}, {it.name}))
97
        sourcesIds = sources.map { it.id }
98
        startWork()
99
100
    }

101
    private fun abortImport() {
102
        isAborted = true
103
104
    }

105
106
    private fun createNotification() {
        if (notification == null) {
107
            notification = NotificationCompat.Builder(this, Notifications.getDnsRuleChannelId(this))
108
109
110
111
112
113
114
115
            notification!!.setSmallIcon(R.drawable.ic_mainnotification)
            notification!!.setOngoing(true)
            notification!!.setAutoCancel(false)
            notification!!.setSound(null)
            notification!!.setOnlyAlertOnce(true)
            notification!!.setUsesChronometer(true)
            notification!!.setContentTitle(getString(R.string.notification_ruleimport_title))
            notification!!.setContentText(getString(R.string.notification_ruleimport_secondarymessage))
116
            notification!!.setStyle(NotificationCompat.BigTextStyle().bigText(getString(R.string.notification_ruleimport_secondarymessage)))
117
            notification!!.setProgress(100, 0, true)
118
            notification!!.setContentIntent(DeepActionState.DNS_RULES.pendingIntentTo(this))
119
120
            val abortPendingAction = PendingIntent.getService(
                this,
121
                RequestCodes.RULE_IMPORT_ABORT,
122
123
124
125
                Intent(this, RuleImportService::class.java).putExtra("abort", true),
                PendingIntent.FLAG_CANCEL_CURRENT
            )
            val abortAction =
126
                NotificationCompat.Action(R.drawable.ic_times, getString(android.R.string.cancel), abortPendingAction)
127
            notification!!.addAction(abortAction)
128
        }
Daniel Wolf's avatar
Daniel Wolf committed
129
        startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
130
131
    }

132
    private fun showSuccessNotification() {
133
        val successNotification = NotificationCompat.Builder(this, Notifications.getDnsRuleChannelId(this))
134
135
136
        successNotification.setSmallIcon(R.drawable.ic_mainnotification)
        successNotification.setAutoCancel(true)
        successNotification.setContentTitle(getString(R.string.notification_ruleimportfinished_title))
137
        getString(R.string.notification_ruleimportfinished_message,
138
139
            getPreferences().numberFormatter.format(ruleCount),
            getPreferences().numberFormatter.format(ruleCount - newlyAddedRuleCount)).apply {
140
141
142
            successNotification.setContentText(this)
            successNotification.setStyle(NotificationCompat.BigTextStyle().bigText(this))
        }
143
        successNotification.setContentIntent(DeepActionState.DNS_RULES.pendingIntentTo(this))
Daniel Wolf's avatar
Daniel Wolf committed
144
        (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager).notify(Notifications.ID_DNSRULE_IMPORT_FINISHED, successNotification.build())
145
146
    }

147
    private fun updateNotification(source: HostSource, count: Int, maxCount: Int) {
148
        if (notification != null) {
149
150
151
152
153
            val notificationText = getString(
                R.string.notification_ruleimport_message,
                source.name,
                source.source
            )
154
            notification?.setContentText(
155
                notificationText
156
            )
157
            notification?.setStyle(NotificationCompat.BigTextStyle().bigText(notificationText))
158
            notification?.setProgress(maxCount, count, false)
Daniel Wolf's avatar
Daniel Wolf committed
159
            startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
160
161
162
        }
    }

163
    private fun updateNotificationFinishing() {
164
        if (notification != null) {
165
166
167
168
169
170
            val notificationText = getString(R.string.notification_ruleimport_tertiarymessage)
            notification?.setContentText(
                notificationText
            )
            notification?.setStyle(NotificationCompat.BigTextStyle().bigText(notificationText))
            notification?.setProgress(1, 1, true)
Daniel Wolf's avatar
Daniel Wolf committed
171
            startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
172
173
174
        }
    }

175
    private fun startWork() {
176
        val dnsRuleDao = getDatabase().dnsRuleDao()
177
        dnsRuleDao.markNonUserRulesForDeletion() // Stage all, ignoring if the source is actually processed in this run
178
        dnsRuleDao.deleteStagedRules()
179
        var count = 0
180
        val newChecksums = mutableMapOf<HostSource, String>()
181
        sources.forEach {
182
183
            log("Importing HostSource $it")
            if (!isAborted) {
184
                updateNotification(it, count, sources.size)
185
186
187
188
189
190
191
                count++
                if (it.isFileSource) {
                    log("Importing from file")
                    var stream: InputStream? = null
                    try {
                        val uri = Uri.parse(it.source)
                        stream = contentResolver.openInputStream(uri)
Daniel Wolf's avatar
Daniel Wolf committed
192
                        processLines(it, stream!!)
193
194
195
196
197
                    } catch (ex: Exception) {
                        log("Import failed: $ex")
                        ex.printStackTrace()
                    } finally {
                        stream?.close()
198
                    }
Daniel Wolf's avatar
Daniel Wolf committed
199
                } else {
200
201
202
203
                    log("Importing from URL")
                    var response: Response? = null
                    try {
                        val request = Request.Builder().url(it.source)
204
205
                        val realChecksum = it.checksum?.replace("<qt>", "\"")
                        if(realChecksum != null) request.header("If-None-Match", realChecksum)
206
                        response = httpClient.newCall(request.build()).execute()
Daniel Wolf's avatar
Daniel Wolf committed
207
208
                        val receivedChecksum = response.headers.find { header ->
                            header.first.equals("etag", true)
209
                        }?.second
210
                        val localDataIsRecent = response.code == 304 || (realChecksum != null && (receivedChecksum == realChecksum || receivedChecksum == "W/$realChecksum"))
211
212
                        when {
                            response.isSuccessful && !localDataIsRecent -> {
Daniel Wolf's avatar
Daniel Wolf committed
213
214
                                response.headers.find { header ->
                                    header.first.equals("etag", true)
215
                                }?.second?.apply {
216
                                    newChecksums[it] = this.replace("\"", "<qt>")
217
                                }
Daniel Wolf's avatar
Daniel Wolf committed
218
                                log("Now parsing content...")
219
220
                                processLines(it, response.body!!.byteStream())
                            }
221
                            localDataIsRecent -> {
222
223
224
225
226
                                log("Host source ${it.name} hasn't changed, not updating.")
                                ruleCount += it.ruleCount ?: 0
                                dnsRuleDao.unstageRulesOfSource(it.id)
                                log("Unstaged rules for ${it.name}")
                            }
227
                            else -> log("Downloading resource of ${it.name} failed (response=${response.code}.")
228
229
                        }
                    } catch (ex: java.lang.Exception) {
230
                        ex.printStackTrace()
231
                        log("Downloading resource of ${it.name} failed ($ex)")
232
                        newChecksums.remove(it)
233
234
235
                    } finally {
                        response?.body?.close()
                    }
Daniel Wolf's avatar
Daniel Wolf committed
236
                }
237
238
239
240
                log("Import of $it finished")
            } else {
                log("Aborting import.")
                return@forEach
241
            }
242
243
244
245
246
        }
        if (!isAborted) {
            updateNotificationFinishing()
            log("Delete rules staged for deletion")
            dnsRuleDao.deleteMarkedRules()
247
            log("Committing staging")
248
            dnsRuleDao.commitStaging()
249
            dnsRuleDao.deleteStagedRules()
250
251
            log("Recreating database indices")
            getDatabase().recreateDnsRuleIndizes()
252
253
254
255
256
257
            log("Updating Etag values for sources")
            newChecksums.forEach { (source, etag) ->
                source.checksum = etag
                getDatabase().hostSourceDao().update(source)
            }
            getDatabase().hostSourceDao().removeChecksumForDisabled()
258
            log("Done.")
259
            if(isServiceRunning(DnsVpnService::class.java)) DnsVpnService.restartVpn(this, false)
Daniel Wolf's avatar
Daniel Wolf committed
260
            newlyAddedRuleCount = sourcesIds.sumBy { dnsRuleDao.getCountForHostSource(it) }
261
262
            showSuccessNotification()
        } else {
263
264
265
            dnsRuleDao.deleteStagedRules()
            dnsRuleDao.commitStaging()
            sendLocalBroadcast(Intent(BROADCAST_IMPORT_DONE))
266
            stopForeground(true)
267
        }
268
269
270
        log("All imports finished.")
        stopForeground(true)
        sendLocalBroadcast(Intent(BROADCAST_IMPORT_DONE))
271
272
    }

273
    private fun processLines(source: HostSource, stream: InputStream) {
274
        val parsers = mutableMapOf(
Daniel Wolf's avatar
Daniel Wolf committed
275
276
277
278
279
            dnsmasqMatcher to (0 to mutableListOf<DnsRule>()),
            hostsMatcher to (0 to mutableListOf()),
            domainsMatcher to (0 to mutableListOf()),
            adblockMatcher to (0 to mutableListOf()),
            dnsmasqBlockMatcher to (0 to mutableListOf())
280
        )
281
        var lineCount = 0
282
        var ruleCount = 0
283
        BufferedReader(InputStreamReader(stream)).useLines { lines ->
284
285
            var remainingMatcher:Matcher? = null
            var hostsOfRemainingMatcher:MutableList<DnsRule>? = null
286
            var successfulMatches = 0
287
288
            lines.forEach { _line ->
                val line = _line.trim()
289
                if (!isAborted) {
Daniel Wolf's avatar
Daniel Wolf committed
290
                    if (parsers.isNotEmpty() && !line.startsWith("#") && !line.startsWith("!") && line.isNotBlank()) {
291
                        lineCount++
292
293
                        if(remainingMatcher != null) {
                            if (remainingMatcher!!.reset(line).matches()) {
294
                                val rule = processLine(remainingMatcher!!, source, source.whitelistSource)
Daniel Wolf's avatar
Daniel Wolf committed
295
                                hostsOfRemainingMatcher!!.add(rule.apply {
Daniel Wolf's avatar
Daniel Wolf committed
296
297
                                    stagingType = 2
                                })
298
                                if (lineCount > ruleCommitSize) {
299
                                    ruleCount += commitLines(parsers)
300
301
                                    lineCount = 0
                                }
302
303
304
305
306
                            }
                        } else {
                            val iterator = parsers.iterator()
                            for ((matcher, hosts) in iterator) {
                                if (matcher.reset(line).matches()) {
307
                                    successfulMatches+=1
308
                                    val rule = processLine(matcher, source, source.whitelistSource)
Daniel Wolf's avatar
Daniel Wolf committed
309
                                    hosts.second.add(rule.apply {
310
311
312
313
314
315
316
                                        stagingType = 2
                                    })
                                    if (lineCount > ruleCommitSize) {
                                        ruleCount += commitLines(parsers)
                                        lineCount = 0
                                    }
                                } else {
317
                                    if(successfulMatches > 35) {
318
319
                                        remainingMatcher = parsers.keys.first()
                                        hostsOfRemainingMatcher = parsers.values.first().second
320
321
322
323
324
325
326
327
328
                                    } else {
                                        if (hosts.first > 5) {
                                            log("Matcher $matcher failed 5 times, last for '$line'. Removing.")
                                            iterator.remove()
                                        } else parsers[matcher] = hosts.copy(hosts.first + 1)
                                        if (parsers.isEmpty()) {
                                            log("No parsers left. Aborting.")
                                            return@forEach
                                        }
329
                                    }
330
                                }
331
                            }
332
333
                        }
                    }
334
335
                } else {
                    return@useLines
336
337
338
                }
            }
        }
339
        if(!isAborted) {
340
341
342
            if (parsers.isNotEmpty()) {
                ruleCount += commitLines(parsers, true)
            }
343
344
345
            source.ruleCount = ruleCount
            getDatabase().hostSourceDao().update(source)
        }
346
347
    }

348
    private fun commitLines(
349
        parsers: Map<Matcher, Pair<Int, MutableList<DnsRule>>>,
350
        forceCommit: Boolean = false
351
    ):Int {
352
        val hosts = parsers[parsers.keys.minByOrNull {
Daniel Wolf's avatar
Daniel Wolf committed
353
354
            parsers[it]?.first!!
        } ?: parsers.keys.first()]?.second!!
355
        return if (hosts.size > ruleCommitSize || forceCommit) {
356
            getDatabase().dnsRuleDao().insertAllIgnoreConflict(hosts)
357
            ruleCount += hosts.size
358
            val added = hosts.size
359
            hosts.clear()
360
361
            added
        } else 0
362
363
    }

364
    private val wwwRegex = Regex("^www\\.")
Daniel Wolf's avatar
Daniel Wolf committed
365
    private fun processLine(matcher: Matcher, source: HostSource, isWhitelist:Boolean): DnsRule {
366
367
        val defaultTargetV4 = if(isWhitelist) "" else "0"
        val defaultTargetV6 = if(isWhitelist) "" else "1"
368
        when {
369
            matcher.groupCount() == 1 -> {
Daniel Wolf's avatar
Daniel Wolf committed
370
                val host = if(matcher == dnsmasqBlockMatcher) "%%" + matcher.group(1)
Daniel Wolf's avatar
Daniel Wolf committed
371
                else matcher.group(1)!!.replace(wwwRegex, "")
372
                return createRule(host, defaultTargetV4, defaultTargetV6, Record.TYPE.ANY, source)
373
            }
Daniel Wolf's avatar
Daniel Wolf committed
374
            matcher == dnsmasqMatcher -> {
Daniel Wolf's avatar
Daniel Wolf committed
375
                val host = "%%" + matcher.group(1)
Daniel Wolf's avatar
Daniel Wolf committed
376
                var target = matcher.group(2)!!
377
378
                val type = if (target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A
                target = target.let {
379
                    when (it) {
380
                        "0.0.0.0", "::", "::0" -> "0"
381
                        "127.0.0.1", "::1" -> "1"
382
383
                        else -> it
                    }
384
                }
385
                return createRule(host, target, null, type, source)
386
            }
Daniel Wolf's avatar
Daniel Wolf committed
387
            matcher == hostsMatcher -> {
388
                return if(isWhitelist) {
Daniel Wolf's avatar
Daniel Wolf committed
389
                    val host = matcher.group(2)!!.replace(wwwRegex, "")
390
                    DnsRule(Record.TYPE.ANY, host, defaultTargetV4, defaultTargetV6, importedFrom = source.id)
391
                } else {
Daniel Wolf's avatar
Daniel Wolf committed
392
                    var target = matcher.group(1)!!
393
394
395
                    val type = if (target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A
                    target = target.let {
                        when (it) {
396
                            "0.0.0.0", "::", "::0" -> "0"
397
398
399
                            "127.0.0.1", "::1" -> "1"
                            else -> it
                        }
400
                    }
Daniel Wolf's avatar
Daniel Wolf committed
401
                    val host = matcher.group(2)!!.replace(wwwRegex, "")
402
                    return createRule(host, target, null, type, source)
403
                }
404
405
406
407
408
            }
        }
        throw IllegalStateException()
    }

Daniel Wolf's avatar
Daniel Wolf committed
409
    private val wildcardNormalisationRegex = Regex("(?:^\\*\\*\\.)|(\\.\\*\\*$)")
410
    private val leadingWildcardRegex = Regex("^\\*\\.")
411
412
413
414
415
416
417
    private fun createRule(
        host: String,
        target: String,
        targetV6: String? = null,
        type: Record.TYPE,
        source: HostSource
    ): DnsRule {
418
419
        var isWildcard = false
        val alteredHost = host.let {
420
            if (it.contains("*")) {
421
                isWildcard = true
422
                if (source.isFileSource) {
423
                    it.replace(leadingWildcardRegex, "**").replace("**", "%%").replace("*", "%")
424
425
                        .replace(wildcardNormalisationRegex, "**")
                } else {
426
                    it.replace(leadingWildcardRegex, "**").replace("**", "%%").replace("*", "%%")
427
428
429
                        .replace(wildcardNormalisationRegex, "**")
                }
            } else it
430
        }
431
        return DnsRule(type, alteredHost, target, targetV6, source.id, isWildcard)
432
433
    }

434
435
    override fun onDestroy() {
        super.onDestroy()
436
        abortImport()
437
438
439
440
441
    }

    override fun onBind(intent: Intent?): IBinder? {
        return null
    }
442
}