RuleImportService.kt 18.1 KB
Newer Older
1
2
package com.frostnerd.smokescreen.service

3
import android.app.IntentService
4
import android.app.NotificationManager
5
import android.app.PendingIntent
6
import android.content.Context
7
import android.content.Intent
Daniel Wolf's avatar
Daniel Wolf committed
8
import android.net.Uri
9
import android.os.IBinder
10
import androidx.core.app.NotificationCompat
11
import com.frostnerd.general.service.isServiceRunning
12
import com.frostnerd.smokescreen.R
13
14
15
import com.frostnerd.smokescreen.database.entities.DnsRule
import com.frostnerd.smokescreen.database.entities.HostSource
import com.frostnerd.smokescreen.database.getDatabase
16
import com.frostnerd.smokescreen.log
17
import com.frostnerd.smokescreen.sendLocalBroadcast
18
import com.frostnerd.smokescreen.util.DeepActionState
19
import com.frostnerd.smokescreen.util.LanguageContextWrapper
20
import com.frostnerd.smokescreen.util.Notifications
21
import com.frostnerd.smokescreen.util.RequestCodes
22
import com.frostnerd.smokescreen.watchIfEnabled
Daniel Wolf's avatar
Daniel Wolf committed
23
import leakcanary.LeakSentry
24
25
import okhttp3.OkHttpClient
import okhttp3.Request
26
import okhttp3.Response
27
import org.minidns.record.Record
Daniel Wolf's avatar
Daniel Wolf committed
28
29
30
import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import java.util.regex.Matcher
import java.util.regex.Pattern

/*
 * Copyright (C) 2019 Daniel Wolf (Ch4t4r)
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.

 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.

 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * You can contact the developer at daniel.wolf@frostnerd.com.
 */
52
class RuleImportService : IntentService("RuleImportService") {
Daniel Wolf's avatar
Daniel Wolf committed
53
    private val dnsmasqMatcher =
54
55
        Pattern.compile("^address=/([^/]+)/(?:([0-9.]+)|([0-9a-fA-F:]+))(?:/?\$|\\s+.*)").matcher("") // address=/xyz.com/0.0.0.0
    private val dnsmasqBlockMatcher = Pattern.compile("^address=/([^/]+)/$").matcher("") // address=/xyz.com/
Daniel Wolf's avatar
Daniel Wolf committed
56
    private val hostsMatcher =
57
58
59
60
        Pattern.compile("^((?:[A-Fa-f0-9:]|[0-9.])+)\\s+([*\\w._\\-]+).*")
            .matcher("") // 0.0.0.0 xyz.com
    private val domainsMatcher = Pattern.compile("^([_\\w*][*\\w_\\-.]+)(?:\$|\\s+.*)").matcher("") // xyz.com
    private val adblockMatcher = Pattern.compile("^\\|\\|(.*)\\^(?:\$|\\s+.*)").matcher("") // ||xyz.com^
61
    private val ruleCommitSize = 10000
62
    private var notification: NotificationCompat.Builder? = null
63
    private var ruleCount: Int = 0
Daniel Wolf's avatar
Daniel Wolf committed
64
    private var newlyAddedRuleCount:Int = 0
65
    private var isAborted = false
66
    private lateinit var sources:List<HostSource>
67
    private lateinit var sourcesIds:List<Long>
68

69
70
71
72
    companion object {
        const val BROADCAST_IMPORT_DONE = "com.frostnerd.nebulo.RULE_IMPORT_DONE"
    }

73
    private val httpClient by lazy(LazyThreadSafetyMode.NONE) {
74
75
76
        OkHttpClient()
    }

77
78
79
80
    override fun attachBaseContext(newBase: Context) {
        super.attachBaseContext(LanguageContextWrapper.attachFromSettings(this, newBase))
    }

Daniel Wolf's avatar
Daniel Wolf committed
81
82
    override fun onCreate() {
        super.onCreate()
83
        LeakSentry.watchIfEnabled(this, "RuleImportService")
84
        (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager).cancel(Notifications.ID_DNSRULE_IMPORT_FINISHED)
Daniel Wolf's avatar
Daniel Wolf committed
85
86
    }

87
    override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
88
        return if (intent != null && intent.hasExtra("abort")) {
89
            abortImport()
90
91
92
93
            START_NOT_STICKY
        } else super.onStartCommand(intent, flags, startId)
    }

Daniel Wolf's avatar
Daniel Wolf committed
94
    override fun onHandleIntent(intent: Intent?) {
95
        createNotification()
96
97
98
99
        sources = (intent?.takeIf { it.hasExtra("sources") }?.getLongArrayExtra("sources")?.let { ids ->
            getDatabase().hostSourceDao().getAllEnabled().filter {
                it.id in ids
            }
100
        } ?: getDatabase().hostSourceDao().getAllEnabled()).sortedWith(compareBy({it.whitelistSource}, {it.name}))
101
        sourcesIds = sources.map { it.id }
102
        startWork()
103
104
    }

105
    private fun abortImport() {
106
        isAborted = true
107
108
    }

109
110
    private fun createNotification() {
        if (notification == null) {
111
            notification = NotificationCompat.Builder(this, Notifications.getDnsRuleChannelId(this))
112
113
114
115
116
117
118
119
            notification!!.setSmallIcon(R.drawable.ic_mainnotification)
            notification!!.setOngoing(true)
            notification!!.setAutoCancel(false)
            notification!!.setSound(null)
            notification!!.setOnlyAlertOnce(true)
            notification!!.setUsesChronometer(true)
            notification!!.setContentTitle(getString(R.string.notification_ruleimport_title))
            notification!!.setContentText(getString(R.string.notification_ruleimport_secondarymessage))
120
            notification!!.setStyle(NotificationCompat.BigTextStyle().bigText(getString(R.string.notification_ruleimport_secondarymessage)))
121
            notification!!.setProgress(100, 0, true)
122
            notification!!.setContentIntent(DeepActionState.DNS_RULES.pendingIntentTo(this))
123
124
            val abortPendingAction = PendingIntent.getService(
                this,
125
                RequestCodes.RULE_IMPORT_ABORT,
126
127
128
129
                Intent(this, RuleImportService::class.java).putExtra("abort", true),
                PendingIntent.FLAG_CANCEL_CURRENT
            )
            val abortAction =
130
                NotificationCompat.Action(R.drawable.ic_times, getString(android.R.string.cancel), abortPendingAction)
131
            notification!!.addAction(abortAction)
132
        }
Daniel Wolf's avatar
Daniel Wolf committed
133
        startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
134
135
    }

136
137
138
139
140
    private fun showSuccessNotification() {
        val successNotification = NotificationCompat.Builder(this, Notifications.getDefaultNotificationChannelId(this))
        successNotification.setSmallIcon(R.drawable.ic_mainnotification)
        successNotification.setAutoCancel(true)
        successNotification.setContentTitle(getString(R.string.notification_ruleimportfinished_title))
141
        getString(R.string.notification_ruleimportfinished_message,
Daniel Wolf's avatar
Daniel Wolf committed
142
143
            ruleCount,
            ruleCount - newlyAddedRuleCount).apply {
144
145
146
            successNotification.setContentText(this)
            successNotification.setStyle(NotificationCompat.BigTextStyle().bigText(this))
        }
147
        successNotification.setContentIntent(DeepActionState.DNS_RULES.pendingIntentTo(this))
Daniel Wolf's avatar
Daniel Wolf committed
148
        (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager).notify(Notifications.ID_DNSRULE_IMPORT_FINISHED, successNotification.build())
149
150
    }

151
    private fun updateNotification(source: HostSource, count: Int, maxCount: Int) {
152
        if (notification != null) {
153
154
155
156
157
            val notificationText = getString(
                R.string.notification_ruleimport_message,
                source.name,
                source.source
            )
158
            notification?.setContentText(
159
                notificationText
160
            )
161
            notification?.setStyle(NotificationCompat.BigTextStyle().bigText(notificationText))
162
            notification?.setProgress(maxCount, count, false)
Daniel Wolf's avatar
Daniel Wolf committed
163
            startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
164
165
166
        }
    }

167
    private fun updateNotificationFinishing() {
168
        if (notification != null) {
169
170
171
172
173
174
            val notificationText = getString(R.string.notification_ruleimport_tertiarymessage)
            notification?.setContentText(
                notificationText
            )
            notification?.setStyle(NotificationCompat.BigTextStyle().bigText(notificationText))
            notification?.setProgress(1, 1, true)
Daniel Wolf's avatar
Daniel Wolf committed
175
            startForeground(Notifications.ID_DNSRULE_IMPORT, notification!!.build())
176
177
178
        }
    }

179
    private fun startWork() {
180
        val dnsRuleDao = getDatabase().dnsRuleDao()
181
        dnsRuleDao.markNonUserRulesForDeletion(sourcesIds) // Stage all, ignoring if the source is actually processed in this run
182
        dnsRuleDao.deleteStagedRules()
183
        var count = 0
184
        val newChecksums = mutableMapOf<HostSource, String>()
185
        sources.forEach {
186
187
            log("Importing HostSource $it")
            if (!isAborted) {
188
                updateNotification(it, count, sources.size)
189
190
191
192
193
194
195
                count++
                if (it.isFileSource) {
                    log("Importing from file")
                    var stream: InputStream? = null
                    try {
                        val uri = Uri.parse(it.source)
                        stream = contentResolver.openInputStream(uri)
Daniel Wolf's avatar
Daniel Wolf committed
196
                        processLines(it, stream!!)
197
198
199
200
201
                    } catch (ex: Exception) {
                        log("Import failed: $ex")
                        ex.printStackTrace()
                    } finally {
                        stream?.close()
202
                    }
Daniel Wolf's avatar
Daniel Wolf committed
203
                } else {
204
205
206
207
                    log("Importing from URL")
                    var response: Response? = null
                    try {
                        val request = Request.Builder().url(it.source)
208
209
                        val realChecksum = it.checksum?.replace("<qt>", "\"")
                        if(realChecksum != null) request.header("If-None-Match", realChecksum)
210
                        response = httpClient.newCall(request.build()).execute()
211
                        val receivedChecksum = response.headers.find {
212
                            it.first.equals("etag", true)
213
                        }?.second
214
                        val localDataIsRecent = response.code == 304 || (realChecksum != null && (receivedChecksum == realChecksum || receivedChecksum == "W/$realChecksum"))
215
216
217
218
219
                        when {
                            response.isSuccessful && !localDataIsRecent -> {
                                response.headers.find {
                                    it.first.equals("etag", true)
                                }?.second?.apply {
220
                                    newChecksums[it] = this.replace("\"", "<qt>")
221
                                }
Daniel Wolf's avatar
Daniel Wolf committed
222
                                log("Now parsing content...")
223
224
                                processLines(it, response.body!!.byteStream())
                            }
225
                            localDataIsRecent -> {
226
227
228
229
230
                                log("Host source ${it.name} hasn't changed, not updating.")
                                ruleCount += it.ruleCount ?: 0
                                dnsRuleDao.unstageRulesOfSource(it.id)
                                log("Unstaged rules for ${it.name}")
                            }
231
                            else -> log("Downloading resource of ${it.name} failed (response=${response.code}.")
232
233
                        }
                    } catch (ex: java.lang.Exception) {
234
                        ex.printStackTrace()
235
                        log("Downloading resource of ${it.name} failed ($ex)")
236
                        newChecksums.remove(it)
237
238
239
                    } finally {
                        response?.body?.close()
                    }
Daniel Wolf's avatar
Daniel Wolf committed
240
                }
241
242
243
244
                log("Import of $it finished")
            } else {
                log("Aborting import.")
                return@forEach
245
            }
246
247
248
249
250
251
252
        }
        if (!isAborted) {
            updateNotificationFinishing()
            log("Delete rules staged for deletion")
            dnsRuleDao.deleteMarkedRules()
            log("Commiting staging")
            dnsRuleDao.commitStaging()
253
            dnsRuleDao.deleteStagedRules()
254
255
            log("Recreating database indices")
            getDatabase().recreateDnsRuleIndizes()
256
257
258
259
260
261
            log("Updating Etag values for sources")
            newChecksums.forEach { (source, etag) ->
                source.checksum = etag
                getDatabase().hostSourceDao().update(source)
            }
            getDatabase().hostSourceDao().removeChecksumForDisabled()
262
            log("Done.")
263
            if(isServiceRunning(DnsVpnService::class.java)) DnsVpnService.restartVpn(this, false)
Daniel Wolf's avatar
Daniel Wolf committed
264
            newlyAddedRuleCount = sourcesIds.sumBy { dnsRuleDao.getCountForHostSource(it) }
265
266
            showSuccessNotification()
        } else {
267
268
269
            dnsRuleDao.deleteStagedRules()
            dnsRuleDao.commitStaging()
            sendLocalBroadcast(Intent(BROADCAST_IMPORT_DONE))
270
            stopForeground(true)
271
        }
272
273
274
        log("All imports finished.")
        stopForeground(true)
        sendLocalBroadcast(Intent(BROADCAST_IMPORT_DONE))
275
276
    }

277
    private fun processLines(source: HostSource, stream: InputStream) {
278
        val parsers = mutableMapOf(
Daniel Wolf's avatar
Daniel Wolf committed
279
280
281
282
283
            dnsmasqMatcher to (0 to mutableListOf<DnsRule>()),
            hostsMatcher to (0 to mutableListOf()),
            domainsMatcher to (0 to mutableListOf()),
            adblockMatcher to (0 to mutableListOf()),
            dnsmasqBlockMatcher to (0 to mutableListOf())
284
        )
285
        var lineCount = 0
286
        var ruleCount = 0
287
        val sourceId = source.id
288
        BufferedReader(InputStreamReader(stream)).useLines { lines ->
289
290
            lines.forEach { _line ->
                val line = _line.trim()
291
                if (!isAborted) {
292
                    if (parsers.isNotEmpty() && !line.startsWith("#") && !line.startsWith("!") && !line.isBlank()) {
293
                        lineCount++
294
295
296
                        val iterator = parsers.iterator()
                        for ((matcher, hosts) in iterator) {
                            if (matcher.reset(line).matches()) {
297
                                val rule = processLine(matcher, sourceId, source.whitelistSource)
298
                                if (rule != null) hosts.second.add(rule.apply {
Daniel Wolf's avatar
Daniel Wolf committed
299
300
                                    stagingType = 2
                                })
301
                                if (lineCount > ruleCommitSize) {
302
                                    ruleCount += commitLines(parsers)
303
304
                                    lineCount = 0
                                }
305
                            } else {
306
                                if (hosts.first > 5) {
307
                                    log("Matcher $matcher failed 5 times, last for '$line'. Removing.")
308
309
                                    iterator.remove()
                                } else parsers[matcher] = hosts.copy(hosts.first + 1)
310
                                if (parsers.isEmpty()) {
311
312
                                    log("No parsers left. Aborting.")
                                }
313
                            }
314
315
                        }
                    }
316
317
                } else {
                    return@useLines
318
319
320
                }
            }
        }
321
        if(!isAborted) {
322
323
324
            if (parsers.isNotEmpty()) {
                ruleCount += commitLines(parsers, true)
            }
325
326
327
            source.ruleCount = ruleCount
            getDatabase().hostSourceDao().update(source)
        }
328
329
    }

330
    private fun commitLines(
331
        parsers: Map<Matcher, Pair<Int, MutableList<DnsRule>>>,
332
        forceCommit: Boolean = false
333
    ):Int {
334
        val hosts = parsers[parsers.keys.minByOrNull {
335
336
            parsers[it]!!.first
        } ?: parsers.keys.first()]!!.second
337
        return if (hosts.size > ruleCommitSize || forceCommit) {
338
            getDatabase().dnsRuleDao().insertAllIgnoreConflict(hosts)
339
            ruleCount += hosts.size
340
            val added = hosts.size
341
            hosts.clear()
342
343
            added
        } else 0
344
345
    }

346
    private val wwwRegex = Regex("^www\\.")
347
348
349
    private fun processLine(matcher: Matcher, sourceId: Long, isWhitelist:Boolean): DnsRule? {
        val defaultTargetV4 = if(isWhitelist) "" else "0"
        val defaultTargetV6 = if(isWhitelist) "" else "1"
350
        when {
351
            matcher.groupCount() == 1 -> {
Daniel Wolf's avatar
Daniel Wolf committed
352
353
                val host = if(matcher == dnsmasqBlockMatcher) "%%" + matcher.group(1)
                else matcher.group(1).replace(wwwRegex, "")
354
                return createRule(host, defaultTargetV4, defaultTargetV6, Record.TYPE.ANY, sourceId)
355
            }
Daniel Wolf's avatar
Daniel Wolf committed
356
            matcher == dnsmasqMatcher -> {
Daniel Wolf's avatar
Daniel Wolf committed
357
                val host = "%%" + matcher.group(1)
358
359
360
                var target = matcher.group(2)
                val type = if (target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A
                target = target.let {
361
362
                    when (it) {
                        "0.0.0.0" -> "0"
363
                        "127.0.0.1", "::1" -> "1"
364
365
                        else -> it
                    }
366
                }
367
                return createRule(host, target, null, type, sourceId)
368
            }
Daniel Wolf's avatar
Daniel Wolf committed
369
            matcher == hostsMatcher -> {
370
371
372
373
374
375
376
377
378
379
380
381
                return if(isWhitelist) {
                    val host = matcher.group(2).replace(wwwRegex, "")
                    DnsRule(Record.TYPE.ANY, host, defaultTargetV4, defaultTargetV6, importedFrom = sourceId)
                } else {
                    var target = matcher.group(1)
                    val type = if (target.contains(":")) Record.TYPE.AAAA else Record.TYPE.A
                    target = target.let {
                        when (it) {
                            "0.0.0.0" -> "0"
                            "127.0.0.1", "::1" -> "1"
                            else -> it
                        }
382
                    }
383
                    val host = matcher.group(2).replace(wwwRegex, "")
384
                    return createRule(host, target, null, type, sourceId)
385
                }
386
387
388
389
390
            }
        }
        throw IllegalStateException()
    }

Daniel Wolf's avatar
Daniel Wolf committed
391
    private val wildcardNormalisationRegex = Regex("(?:^\\*\\*\\.)|(\\.\\*\\*$)")
392
393
394
395
396
    private fun createRule(host: String, target: String, targetV6:String? = null, type: Record.TYPE, sourceId: Long): DnsRule? {
        var isWildcard = false
        val alteredHost = host.let {
            if(it.contains("*")) {
                isWildcard = true
Daniel Wolf's avatar
Daniel Wolf committed
397
398
                it.replace("**", "%%").replace("*", "%").replace(wildcardNormalisationRegex, "**")
        } else it
399
400
        }
        return DnsRule(type, alteredHost, target, targetV6, sourceId, isWildcard)
401
402
    }

403
404
    override fun onDestroy() {
        super.onDestroy()
405
        abortImport()
406
407
408
409
410
411
    }

    override fun onBind(intent: Intent?): IBinder? {
        return null
    }
}