adblock-fast: update to 1.2.2-r16

* add: ucode-mod-uloop dependency
* add: parallel downloads using uloop
* fix: explicit allow for domains from allow-lists
* fix: get environment information for getInitStatus RPCD call
* add: update tests

Signed-off-by: Stan Grishin <stangri@melmac.ca>

net/adblock-fast/Makefile

@@ -5,7 +5,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock-fast
 PKG_VERSION:=1.2.2
-PKG_RELEASE:=14
+PKG_RELEASE:=16
 PKG_MAINTAINER:=Stan Grishin <stangri@melmac.ca>
 PKG_LICENSE:=AGPL-3.0-or-later
 
@@ -24,6 +24,7 @@ define Package/adblock-fast
 	+ucode-mod-fs \
 	+ucode-mod-uci \
 	+ucode-mod-ubus \
+	+ucode-mod-uloop \
 	+!BUSYBOX_DEFAULT_AWK:gawk \
 	+!BUSYBOX_DEFAULT_GREP:grep \
 	+!BUSYBOX_DEFAULT_SED:sed \

net/adblock-fast/files/lib/adblock-fast/adblock-fast.uc

@@ -8,6 +8,7 @@
 import { readfile, writefile, popen, stat, unlink, rename, open, glob, mkdir, mkstemp, symlink, chmod, chown, realpath, lsdir, access, dirname } from 'fs';
 import { cursor } from 'uci';
 import { connect } from 'ubus';
+import * as uloop from 'uloop';
 
 // ── Constants ───────────────────────────────────────────────────────
 
@@ -1553,7 +1554,7 @@ function resolver(action) {
 
 // ── process_file_url ────────────────────────────────────────────────
 
-function process_file_url(section, url_override, action_override) {
+function process_file_url(section, url_override, action_override, predownloaded) {
 	let url, file_action, name, size_val;
 
 	if (section && !url_override) {
@@ -1585,15 +1586,22 @@ function process_file_url(section, url_override, action_override) {
 	case 'file': type_name = 'File'; d_tmp = tmp.b; break;
 	}
 
-	if (is_https_url(url) && !env.get_downloader().ssl_supported) {
+	if (!predownloaded && is_https_url(url) && !env.get_downloader().ssl_supported) {
 		output.info(sym.fail[0]);
 		output.verbose('[ DL ] ' + type_name + ' ' + label + ' ' + sym.fail[1] + '\\n');
 		push(status_data.errors, { code: 'errorNoSSLSupport', info: name || url });
 		return true;
 	}
 
-	let r_tmp = trim(cmd_output('mktemp -q -t "' + pkg.name + '_tmp.XXXXXXXX"'));
-	if (!url || !download(url, r_tmp) || !(stat(r_tmp)?.size > 0)) {
+	let r_tmp = predownloaded || trim(cmd_output('mktemp -q -t "' + pkg.name + '_tmp.XXXXXXXX"'));
+	if (predownloaded && !(stat(r_tmp)?.size > 0)) {
+		output.info(sym.fail[0]);
+		output.verbose('[ DL ] ' + type_name + ' ' + label + ' ' + sym.fail[1] + '\\n');
+		push(status_data.errors, { code: 'errorDownloadingList', info: name || url });
+		unlink(r_tmp);
+		return true;
+	}
+	if (!predownloaded && (!url || !download(url, r_tmp) || !(stat(r_tmp)?.size > 0))) {
 		output.info(sym.fail[0]);
 		output.verbose('[ DL ] ' + type_name + ' ' + label + ' ' + sym.fail[1] + '\\n');
 		push(status_data.errors, { code: 'errorDownloadingList', info: name || url });
@@ -1710,8 +1718,44 @@ function download_lists() {
 	let download_cfgs = [];
 	uci(pkg.name).foreach(pkg.name, 'file_url', (s) => push(download_cfgs, s['.name']));
 
+	if (cfg.parallel_downloads && uloop && length(download_cfgs) > 1) {
+		// Parallel mode: download all files first, then process each
+		let dlt = env.get_downloader();
+		let jobs = [];
+		for (let cfg_name in download_cfgs) {
+			let sec_cur = cursor();
+			sec_cur.load(pkg.name);
+			if (sec_cur.get(pkg.name, cfg_name, 'enabled') == '0') continue;
+			let url = sec_cur.get(pkg.name, cfg_name, 'url');
+			if (!url) continue;
+			if (is_https_url(url) && !dlt.ssl_supported) {
+				let name = sec_cur.get(pkg.name, cfg_name, 'name');
+				push(status_data.errors, { code: 'errorNoSSLSupport', info: name || url });
+				output.info(sym.fail[0]);
+				continue;
+			}
+			let r_tmp = trim(cmd_output('mktemp -q -t "' + pkg.name + '_tmp.XXXXXXXX"'));
+			push(jobs, { cfg_name, url, r_tmp });
+		}
+		if (length(jobs) > 0) {
+			uloop.init();
+			let pending = length(jobs);
+			for (let job in jobs) {
+				let dl_cmd = sprintf('%s %s %s %s 2>/dev/null',
+					dlt.command, shell_quote(job.url), dlt.flag, shell_quote(job.r_tmp));
+				uloop.process('/bin/sh', ['-c', dl_cmd], {}, () => {
+					if (--pending == 0) uloop.end();
+				});
+			}
+			uloop.run();
+			uloop.done();
+			for (let job in jobs)
+				process_file_url(job.cfg_name, null, null, job.r_tmp);
+		}
+	} else {
 		for (let cfg_name in download_cfgs)
 			process_file_url(cfg_name);
+	}
 
 	if (uci_has_changes(pkg.name)) {
 		output.verbose('[PROC] Saving updated file sizes ');
@@ -1856,14 +1900,18 @@ function download_lists() {
 	logger_debug('[PERF-DEBUG] ' + step_title + ' took ' + elapsed + 's');
 
 	// Explicitly allow domains in servers mode
-	if (dns_output.allow_filter && cfg.allowed_domain) {
+	if (dns_output.allow_filter && (cfg.allowed_domain || (stat(tmp.allowed)?.size > 0))) {
 		unlink(tmp.sed); writefile(tmp.sed, '');
 		start_time = time();
 		step_title = 'Explicitly allowing domains in ' + cfg.dns;
 		output.verbose('[PROC] ' + step_title + ' ');
 		status_data.message = get_text('statusProcessing') + ': ' + step_title;
+		let allowed_list_extra = '';
+		if (stat(tmp.allowed)?.size > 0)
+			allowed_list_extra = trim(cmd_output(sprintf("sed '/^[[:space:]]*$/d' %s", shell_quote(tmp.allowed))));
+		let all_allow = (cfg.allowed_domain || '') + (allowed_list_extra ? ' ' + allowed_list_extra : '');
 		let allow_input = '';
-		for (let hf in split('' + cfg.allowed_domain, /\s+/))
+		for (let hf in split(all_allow, /\s+/))
 			if (hf) allow_input += hf + '\n';
 		if (allow_input)
 			system(sprintf("printf '%%s' %s | sed -E '%s' >> %s", shell_quote(allow_input), dns_output.allow_filter, shell_quote(tmp.sed)));
@@ -2714,6 +2762,7 @@ function get_network_trigger_info() {
 function get_init_status(name) {
 	name = name || pkg.name;
 	env.load_config();
+	env.detect();
 
 	// Read pre-computed data from procd service (like PBR)
 	let conn = connect();

net/adblock-fast/tests/01_pipeline/10_servers_mode_allow_list

@@ -0,0 +1,117 @@
+Test that dnsmasq.servers mode prepends explicit allow entries (server=/domain/#)
+for domains from allowed list files, not just config-defined allowed_domain.
+
+-- File uci/adblock-fast.json --
+{
+	"config": [
+		{
+			".name": "config",
+			".type": "config",
+			"enabled": "1",
+			"dns": "dnsmasq.servers",
+			"verbosity": "0",
+			"force_dns": "0",
+			"compressed_cache": "0",
+			"config_update_enabled": "0",
+			"ipv6_enabled": "0",
+			"canary_domains_icloud": "0",
+			"canary_domains_mozilla": "0",
+			"dnsmasq_sanity_check": "0",
+			"dnsmasq_validity_check": "0",
+			"parallel_downloads": "0",
+			"allow_non_ascii": "0",
+			"update_config_sizes": "0",
+			"heartbeat_domain": "-",
+			"download_timeout": "10",
+			"pause_timeout": "20",
+			"curl_retry": "1",
+			"compressed_cache_dir": "TESTDIR/cache",
+			"allowed_domain": "config-allowed.example.com"
+		}
+	],
+	"file_url": [
+		{
+			".name": "blocked_domains",
+			".type": "file_url",
+			"enabled": "1",
+			"url": "file://TESTDIR/data/domains.txt",
+			"action": "block",
+			"name": "Test Domains"
+		},
+		{
+			".name": "blocked_hosts",
+			".type": "file_url",
+			"enabled": "1",
+			"url": "file://TESTDIR/data/hosts.txt",
+			"action": "block",
+			"name": "Test Hosts"
+		},
+		{
+			".name": "allowed_list",
+			".type": "file_url",
+			"enabled": "1",
+			"url": "file://TESTDIR/data/allowed.txt",
+			"action": "allow",
+			"name": "Test Allowed"
+		}
+	]
+}
+-- End --
+
+-- Testcase --
+import adb from 'adblock-fast';
+import { readfile } from 'fs';
+let ti = adb._test_internals;
+
+adb.env.load_config();
+ti.set_cfg('dns', 'dnsmasq.servers');
+ti.set_cfg('dnsmasq_sanity_check', false);
+ti.set_cfg('dnsmasq_validity_check', false);
+ti.set_cfg('heartbeat_domain', null);
+ti.set_cfg('config_update_enabled', false);
+ti.set_cfg('update_config_sizes', false);
+ti.env.dns_set_output_values('dnsmasq.servers');
+ti.append_urls();
+
+let ok = ti.download_lists();
+if (!ok) {
+	print('download_lists failed\n');
+} else {
+	let content = readfile(ti.dns_output.file) || '';
+	let lines = filter(split(content, '\n'), l => length(l) > 0);
+
+	let allow_entries = filter(lines, l => match(l, /\/#$/));
+
+	let results = [];
+
+	// Check config-defined allowed domain
+	let has_config = index(content, 'server=/config-allowed.example.com/#') >= 0;
+	push(results, sprintf('config-allowed.example.com allow: %s', has_config ? 'PRESENT' : 'MISSING'));
+
+	// Check list-file allowed domains (from allowed.txt which has hosts format)
+	let has_list1 = index(content, 'server=/adhost-zero-1.test.example.org/#') >= 0;
+	let has_list2 = index(content, 'server=/common-shared-1.test.example.com/#') >= 0;
+	push(results, sprintf('adhost-zero-1.test.example.org allow: %s', has_list1 ? 'PRESENT' : 'MISSING'));
+	push(results, sprintf('common-shared-1.test.example.com allow: %s', has_list2 ? 'PRESENT' : 'MISSING'));
+
+	// Allow entries should be at the top
+	if (length(allow_entries) > 0 && length(lines) > 0) {
+		let first_is_allow = match(lines[0], /\/#$/);
+		push(results, sprintf('allow_entries_at_top: %s', first_is_allow ? 'YES' : 'NO'));
+	}
+
+	// Allowed domains should NOT appear as block entries
+	let blocked_list1 = index(content, 'server=/adhost-zero-1.test.example.org/\n') >= 0;
+	push(results, sprintf('adhost-zero-1.test.example.org blocked: %s', blocked_list1 ? 'STILL PRESENT (BAD)' : 'REMOVED'));
+
+	print(join('\n', results) + '\n');
+}
+-- End --
+
+-- Expect stdout --
+config-allowed.example.com allow: PRESENT
+adhost-zero-1.test.example.org allow: PRESENT
+common-shared-1.test.example.com allow: PRESENT
+allow_entries_at_top: YES
+adhost-zero-1.test.example.org blocked: REMOVED
+-- End --

net/adblock-fast/tests/run_tests.sh

@@ -88,6 +88,7 @@ sed \
 	-e "s|import { readfile, writefile, popen, stat, unlink, rename, open, glob, mkdir, mkstemp, symlink, chmod, chown, realpath, lsdir, access, dirname } from 'fs';|let _fs = require('fs'), readfile = _fs.readfile, writefile = _fs.writefile, popen = _fs.popen, stat = _fs.stat, unlink = _fs.unlink, rename = _fs.rename, open = _fs.open, glob = _fs.glob, mkdir = _fs.mkdir, mkstemp = _fs.mkstemp, symlink = _fs.symlink, chmod = _fs.chmod, chown = _fs.chown, realpath = _fs.realpath, lsdir = _fs.lsdir, access = _fs.access, dirname = _fs.dirname;|" \
 	-e "s|import { cursor } from 'uci';|let _uci = require('uci'), cursor = _uci.cursor;|" \
 	-e "s|import { connect } from 'ubus';|let _ubus = require('ubus'), connect = _ubus.connect;|" \
+	-e "s|import \* as uloop from 'uloop';|let uloop = null;|" \
 	-e "s|dnsmasq_file: '/var/run/adblock-fast/adblock-fast.dnsmasq'|dnsmasq_file: '${TESTDIR}/var_run/adblock-fast/adblock-fast.dnsmasq'|" \
 	-e "s|config_file: '/etc/config/adblock-fast'|config_file: '${TESTDIR}/etc/adblock-fast'|" \
 	-e "s|run_file: '/dev/shm/adblock-fast'|run_file: '${TESTDIR}/shm/adblock-fast'|" \