RuleImportService.kt 18.6 KB
Newer Older
1
2
package com.frostnerd.smokescreen.service

3
import android.app.IntentService
4
import android.app.NotificationManager
5
import android.app.PendingIntent
6
import android.content.Context
7
import android.content.Intent
Daniel Wolf's avatar
Daniel Wolf committed
8
import android.net.Uri
9
import android.os.IBinder
10
import androidx.core.app.NotificationCompat
11
import com.frostnerd.general.service.isServiceRunning
12
import com.frostnerd.smokescreen.R
13
14
15
import com.frostnerd.smokescreen.database.entities.DnsRule
import com.frostnerd.smokescreen.database.entities.HostSource
import com.frostnerd.smokescreen.database.getDatabase
16
import com.frostnerd.smokescreen.log
17
import com.frostnerd.smokescreen.sendLocalBroadcast
18
import com.frostnerd.smokescreen.util.DeepActionState
19
import com.frostnerd.smokescreen.util.LanguageContextWrapper
20
import com.frostnerd.smokescreen.util.Notifications
21
import com.frostnerd.smokescreen.util.RequestCodes
22
import com.frostnerd.smokescreen.watchIfEnabled
Daniel Wolf's avatar
Daniel Wolf committed
23
import leakcanary.LeakSentry
24
25
import okhttp3.OkHttpClient
import okhttp3.Request
26
import okhttp3.Response
27
import org.minidns.record.Record
Daniel Wolf's avatar
Daniel Wolf committed
28
29
30
import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import java.util.regex.Matcher
import java.util.regex.Pattern

/*
 * Copyright (C) 2019 Daniel Wolf (Ch4t4r)
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.

 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.

 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * You can contact the developer at daniel.wolf@frostnerd.com.
 */
52
class RuleImportService : IntentService("RuleImportService") {
Daniel Wolf's avatar
Daniel Wolf committed
53
    private val dnsmasqMatcher =
54
55
        Pattern.compile("^address=/([^/]+)/(?:([0-9.]+)|([0-9a-fA-F:]+))(?:/?\$|\\s+.*)").matcher("") // address=/xyz.com/0.0.0.0
    private val dnsmasqBlockMatcher = Pattern.compile("^address=/([^/]+)/$").matcher("") // address=/xyz.com/
Daniel Wolf's avatar
Daniel Wolf committed
56
    private val hostsMatcher =
57
58
59
60
        Pattern.compile("^((?:[A-Fa-f0-9:]|[0-9.])+)\\s+([*\\w._\\-]+).*")
            .matcher("") // 0.0.0.0 xyz.com
    private val domainsMatcher = Pattern.compile("^([_\\w*][*\\w_\\-.]+)(?:\$|\\s+.*)").matcher("") // xyz.com
    private val adblockMatcher = Pattern.compile("^\\|\\|(.*)\\^(?:\$|\\s+.*)").matcher("") // ||xyz.com^
61
    private val ruleCommitSize = 10000
62
    private var notification: NotificationCompat.Builder? = null
63
    private var ruleCount: Int = 0
Daniel Wolf's avatar
Daniel Wolf committed
64
    private var newlyAddedRuleCount:Int = 0
65
    private var isAborted = false
66
    private lateinit var sources:List<HostSource>
67
    private lateinit var sourcesIds:List<Long>
68

69
70
71
72
    companion object {
        const val BROADCAST_IMPORT_DONE = "com.frostnerd.nebulo.RULE_IMPORT_DONE"
    }

73
    private val httpClient by lazy(LazyThreadSafetyMode.NONE) {
74
75
76
        OkHttpClient()
    }

77
78
79
80
    override fun attachBaseContext(newBase: Context) {
        super.attachBaseContext(LanguageContextWrapper.attachFromSettings(this, newBase))
    }

Daniel Wolf's avatar
Daniel Wolf committed
81
82
    override fun onCreate() {
        super.onCreate()
83
        LeakSentry.watchIfEnabled(this, "RuleImportService")
84
        (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager).cancel(Notifications.ID_DNSRULE_IMPORT_FINISHED)
Daniel Wolf's avatar
Daniel Wolf committed
85
86
    }

87
    override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
88
        return if (intent != null && intent.hasExtra("abort")) {
89
            abortImport()
90
91
92
93
            START_NOT_STICKY
        } else super.onStartCommand(intent, flags, startId)
    }

Daniel Wolf's avatar
Daniel Wolf committed
94
    override fun onHandleIntent(intent: Intent?) {
95
        createNotification()
96
97
98
99
        sources = (intent?.takeIf { it.hasExtra("sources") }?.getLongArrayExtra("sources")?.let { ids ->
            getDatabase().hostSourceDao().getAllEnabled().filter {
                it.id in ids
            }
100
        } ?: getDatabase().hostSourceDao().getAllEnabled()).sortedWith(compareBy({it.whitelistSource}, {it.name}))
101
        sourcesIds = sources.map { it.id }
102
        startWork()
103
104
    }

105
    private fun abortImport() {
106
        isAborted = true
107
108
    }

109
110
    private fun createNotification() {
        if (notification == null) {
111
            notification = NotificationCompat.Builder(this, Notifications.getDnsRuleChannelId(this))
112
113
114
115
116
117
118
119
            notification!!.setSmallIcon(R.drawable.ic_mainnotification)
            notification!!.setOngoing(true)
            notification!!.setAutoCancel(false)
            notification!!.setSound(null)
            notification!!.setOnlyAlertOnce(true)
            notification!!.setUsesChronometer(true)
            notification!!.setContentTitle(getString(R.string.notification_ruleimport_title))
            notification!!.setContentText(getString(R.string.notification_ruleimport_secondarymessage))
120
            notification!!.setStyle(NotificationCompat.BigTextStyle().bigText(getString(R.string.notification_ruleimport_secondarymessage)))
121
            notification!!.setProgress(100, 0, true)
122
            notification!!.setContentIntent(DeepActionState.DNS_RULES.pendingIntentTo(this))
123
124
            val abortPendingAction = PendingIntent.getService(
                this,
125
                RequestCodes.RULE_IMPORT_ABORT,
126
127
128
129
                Intent(this, RuleImportService::class.java).putExtra("abort", true),
                PendingIntent.FLAG_CANCEL_CURRENT
            )
            val abortAction =
130
                NotificationCompat.Action(R.drawable.ic_times, getString(android.R.string.cancel), abortPendingAction)
131
            notification!!.addAction(abortAction)
132
        }
Daniel Wolf's avatar
Daniel Wolf committed
133
        startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
134
135
    }

136
137
138
139
140
    private fun showSuccessNotification() {
        val successNotification = NotificationCompat.Builder(this, Notifications.getDefaultNotificationChannelId(this))
        successNotification.setSmallIcon(R.drawable.ic_mainnotification)
        successNotification.setAutoCancel(true)
        successNotification.setContentTitle(getString(R.string.notification_ruleimportfinished_title))
141
        getString(R.string.notification_ruleimportfinished_message,
Daniel Wolf's avatar
Daniel Wolf committed
142
143
            ruleCount,
            ruleCount - newlyAddedRuleCount).apply {
144
145
146
            successNotification.setContentText(this)
            successNotification.setStyle(NotificationCompat.BigTextStyle().bigText(this))
        }
147
        successNotification.setContentIntent(DeepActionState.DNS_RULES.pendingIntentTo(this))
Daniel Wolf's avatar
Daniel Wolf committed
148
        (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager).notify(Notifications.ID_DNSRULE_IMPORT_FINISHED, successNotification.build())
149
150
    }

151
    private fun updateNotification(source: HostSource, count: Int, maxCount: Int) {
152
        if (notification != null) {
153
154
155
156
157
            val notificationText = getString(
                R.string.notification_ruleimport_message,
                source.name,
                source.source
            )
158
            notification?.setContentText(
159
                notificationText
160
            )
161
            notification?.setStyle(NotificationCompat.BigTextStyle().bigText(notificationText))
162
            notification?.setProgress(maxCount, count, false)
Daniel Wolf's avatar
Daniel Wolf committed
163
            startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
164
165
166
        }
    }

167
    private fun updateNotificationFinishing() {
168
        if (notification != null) {
169
170
171
172
173
174
            val notificationText = getString(R.string.notification_ruleimport_tertiarymessage)
            notification?.setContentText(
                notificationText
            )
            notification?.setStyle(NotificationCompat.BigTextStyle().bigText(notificationText))
            notification?.setProgress(1, 1, true)
Daniel Wolf's avatar
Daniel Wolf committed
175
            startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
176
177
178
        }
    }

179
    private fun startWork() {
180
        val dnsRuleDao = getDatabase().dnsRuleDao()
181
        dnsRuleDao.markNonUserRulesForDeletion(sourcesIds) // Stage all, ignoring if the source is actually processed in this run
182
        dnsRuleDao.deleteStagedRules()
183
        var count = 0
184
        val newChecksums = mutableMapOf<HostSource, String>()
185
        sources.forEach {
186
187
            log("Importing HostSource $it")
            if (!isAborted) {
188
                updateNotification(it, count, sources.size)
189
190
191
192
193
194
195
                count++
                if (it.isFileSource) {
                    log("Importing from file")
                    var stream: InputStream? = null
                    try {
                        val uri = Uri.parse(it.source)
                        stream = contentResolver.openInputStream(uri)
Daniel Wolf's avatar
Daniel Wolf committed
196
                        processLines(it, stream!!)
197
198
199
200
201
                    } catch (ex: Exception) {
                        log("Import failed: $ex")
                        ex.printStackTrace()
                    } finally {
                        stream?.close()
202
                    }
Daniel Wolf's avatar
Daniel Wolf committed
203
                } else {
204
205
206
207
                    log("Importing from URL")
                    var response: Response? = null
                    try {
                        val request = Request.Builder().url(it.source)
208
209
                        val realChecksum = it.checksum?.replace("<qt>", "\"")
                        if(realChecksum != null) request.header("If-None-Match", realChecksum)
210
                        response = httpClient.newCall(request.build()).execute()
211
                        val receivedChecksum = response.headers.find {
212
                            it.first.equals("etag", true)
213
                        }?.second
214
                        val localDataIsRecent = response.code == 304 || (realChecksum != null && (receivedChecksum == realChecksum || receivedChecksum == "W/$realChecksum"))
215
216
217
218
219
                        when {
                            response.isSuccessful && !localDataIsRecent -> {
                                response.headers.find {
                                    it.first.equals("etag", true)
                                }?.second?.apply {
220
                                    newChecksums[it] = this.replace("\"", "<qt>")
221
                                }
Daniel Wolf's avatar
Daniel Wolf committed
222
                                log("Now parsing content...")
223
224
                                processLines(it, response.body!!.byteStream())
                            }
225
                            localDataIsRecent -> {
226
227
228
229
230
                                log("Host source ${it.name} hasn't changed, not updating.")
                                ruleCount += it.ruleCount ?: 0
                                dnsRuleDao.unstageRulesOfSource(it.id)
                                log("Unstaged rules for ${it.name}")
                            }
231
                            else -> log("Downloading resource of ${it.name} failed (response=${response.code}.")
232
233
                        }
                    } catch (ex: java.lang.Exception) {
234
                        ex.printStackTrace()
235
                        log("Downloading resource of ${it.name} failed ($ex)")
236
                        newChecksums.remove(it)
237
238
239
                    } finally {
                        response?.body?.close()
                    }
Daniel Wolf's avatar
Daniel Wolf committed
240
                }
241
242
243
244
                log("Import of $it finished")
            } else {
                log("Aborting import.")
                return@forEach
245
            }
246
247
248
249
250
251
252
        }
        if (!isAborted) {
            updateNotificationFinishing()
            log("Delete rules staged for deletion")
            dnsRuleDao.deleteMarkedRules()
            log("Commiting staging")
            dnsRuleDao.commitStaging()
253
            dnsRuleDao.deleteStagedRules()
254
255
            log("Recreating database indices")
            getDatabase().recreateDnsRuleIndizes()
256
257
258
259
260
261
            log("Updating Etag values for sources")
            newChecksums.forEach { (source, etag) ->
                source.checksum = etag
                getDatabase().hostSourceDao().update(source)
            }
            getDatabase().hostSourceDao().removeChecksumForDisabled()
262
            log("Done.")
263
            if(isServiceRunning(DnsVpnService::class.java)) DnsVpnService.restartVpn(this, false)
Daniel Wolf's avatar
Daniel Wolf committed
264
            newlyAddedRuleCount = sourcesIds.sumBy { dnsRuleDao.getCountForHostSource(it) }
265
266
            showSuccessNotification()
        } else {
267
268
269
            dnsRuleDao.deleteStagedRules()
            dnsRuleDao.commitStaging()
            sendLocalBroadcast(Intent(BROADCAST_IMPORT_DONE))
270
            stopForeground(true)
271
        }
272
273
274
        log("All imports finished.")
        stopForeground(true)
        sendLocalBroadcast(Intent(BROADCAST_IMPORT_DONE))
275
276
    }

277
    private fun processLines(source: HostSource, stream: InputStream) {
278
        val parsers = mutableMapOf(
Daniel Wolf's avatar
Daniel Wolf committed
279
280
281
282
283
            dnsmasqMatcher to (0 to mutableListOf<DnsRule>()),
            hostsMatcher to (0 to mutableListOf()),
            domainsMatcher to (0 to mutableListOf()),
            adblockMatcher to (0 to mutableListOf()),
            dnsmasqBlockMatcher to (0 to mutableListOf())
284
        )
285
        var lineCount = 0
286
        var ruleCount = 0
287
        val sourceId = source.id
288
        BufferedReader(InputStreamReader(stream)).useLines { lines ->
289
            var validParsings = 0
290
291
            lines.forEach { _line ->
                val line = _line.trim()
292
                if (!isAborted) {
293
                    if (parsers.isNotEmpty() && !line.startsWith("#") && !line.startsWith("!") && !line.isBlank()) {
294
                        lineCount++
295
296
297
                        val iterator = parsers.iterator()
                        for ((matcher, hosts) in iterator) {
                            if (matcher.reset(line).matches()) {
298
                                validParsings++
299
                                val rule = processLine(matcher, sourceId, source.whitelistSource)
300
                                if (rule != null) hosts.second.add(rule.apply {
Daniel Wolf's avatar
Daniel Wolf committed
301
302
                                    stagingType = 2
                                })
303
                                if (lineCount > ruleCommitSize) {
304
                                    ruleCount += commitLines(parsers)
305
306
                                    lineCount = 0
                                }
307
                            } else {
308
309
310
311
312
313
314
315
316
317
                                // If validParsings is at least 26 we know for sure that the current active parser had at least 5 successful hits
                                // So we are going to keep it for the rest of the document, even if it fails
                                if(validParsings <= 25) {
                                    if (hosts.first > 5) {
                                        log("Matcher $matcher failed 5 times, last for '$line'. Removing.")
                                        iterator.remove()
                                    } else parsers[matcher] = hosts.copy(hosts.first + 1)
                                    if (parsers.isEmpty()) {
                                        log("No parsers left. Aborting.")
                                    }
318
                                }
319
                            }
320
321
                        }
                    }
322
323
                } else {
                    return@useLines
324
325
326
                }
            }
        }
327
        if(!isAborted) {
328
329
330
            if (parsers.isNotEmpty()) {
                ruleCount += commitLines(parsers, true)
            }
331
332
333
            source.ruleCount = ruleCount
            getDatabase().hostSourceDao().update(source)
        }
334
335
    }

336
    private fun commitLines(
337
        parsers: Map<Matcher, Pair<Int, MutableList<DnsRule>>>,
338
        forceCommit: Boolean = false
339
    ):Int {
340
        val hosts = parsers[parsers.keys.minByOrNull {
341
342
            parsers[it]!!.first
        } ?: parsers.keys.first()]!!.second
343
        return if (hosts.size > ruleCommitSize || forceCommit) {
344
            getDatabase().dnsRuleDao().insertAllIgnoreConflict(hosts)
345
            ruleCount += hosts.size
346
            val added = hosts.size
347
            hosts.clear()
348
349
            added
        } else 0
350
351
    }

352
    private val wwwRegex = Regex("^www\\.")
353
354
355
    private fun processLine(matcher: Matcher, sourceId: Long, isWhitelist:Boolean): DnsRule? {
        val defaultTargetV4 = if(isWhitelist) "" else "0"
        val defaultTargetV6 = if(isWhitelist) "" else "1"
356
        when {
357
            matcher.groupCount() == 1 -> {
Daniel Wolf's avatar
Daniel Wolf committed
358
359
                val host = if(matcher == dnsmasqBlockMatcher) "%%" + matcher.group(1)
                else matcher.group(1).replace(wwwRegex, "")
360
                return createRule(host, defaultTargetV4, defaultTargetV6, Record.TYPE.ANY, sourceId)
361
            }
Daniel Wolf's avatar
Daniel Wolf committed
362
            matcher == dnsmasqMatcher -> {
Daniel Wolf's avatar
Daniel Wolf committed
363
                val host = "%%" + matcher.group(1)
364
365
366
                var target = matcher.group(2)
                val type = if (target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A
                target = target.let {
367
368
                    when (it) {
                        "0.0.0.0" -> "0"
369
                        "127.0.0.1", "::1" -> "1"
370
371
                        else -> it
                    }
372
                }
373
                return createRule(host, target, null, type, sourceId)
374
            }
Daniel Wolf's avatar
Daniel Wolf committed
375
            matcher == hostsMatcher -> {
376
377
378
379
380
381
382
383
384
385
386
387
                return if(isWhitelist) {
                    val host = matcher.group(2).replace(wwwRegex, "")
                    DnsRule(Record.TYPE.ANY, host, defaultTargetV4, defaultTargetV6, importedFrom = sourceId)
                } else {
                    var target = matcher.group(1)
                    val type = if (target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A
                    target = target.let {
                        when (it) {
                            "0.0.0.0" -> "0"
                            "127.0.0.1", "::1" -> "1"
                            else -> it
                        }
388
                    }
389
                    val host = matcher.group(2).replace(wwwRegex, "")
390
                    return createRule(host, target, null, type, sourceId)
391
                }
392
393
394
395
396
            }
        }
        throw IllegalStateException()
    }

Daniel Wolf's avatar
Daniel Wolf committed
397
    private val wildcardNormalisationRegex = Regex("(?:^\\*\\*\\.)|(\\.\\*\\*$)")
398
399
400
401
402
    private fun createRule(host: String, target: String, targetV6:String? = null, type: Record.TYPE, sourceId: Long): DnsRule? {
        var isWildcard = false
        val alteredHost = host.let {
            if(it.contains("*")) {
                isWildcard = true
Daniel Wolf's avatar
Daniel Wolf committed
403
404
                it.replace("**", "%%").replace("*", "%").replace(wildcardNormalisationRegex, "**")
        } else it
405
406
        }
        return DnsRule(type, alteredHost, target, targetV6, sourceId, isWildcard)
407
408
    }

409
410
    override fun onDestroy() {
        super.onDestroy()
411
        abortImport()
412
413
414
415
416
417
    }

    override fun onBind(intent: Intent?): IBinder? {
        return null
    }
}