From dc39393c1b9668f0c92560e6566ee64709a6abf6 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Mon, 18 May 2026 23:21:19 +0200 Subject: [PATCH] banip: update 1.8.8-4 - f_etag performance optimization: single-pass awk consolidating count+match - f_report performance optimization: significantly reduce subshell spawning - f_lookup performance optimization: DNS resolution parallelized per domain via subshells - LuCI: prevent possible report refresh timeouts Signed-off-by: Dirk Brenken --- net/banip/Makefile | 2 +- net/banip/files/banip-functions.sh | 160 +++++++++++++++++++---------- net/banip/files/banip-service.sh | 2 +- 3 files changed, 108 insertions(+), 56 deletions(-) diff --git a/net/banip/Makefile b/net/banip/Makefile index ad2c38723f..41af8f8938 100644 --- a/net/banip/Makefile +++ b/net/banip/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=banip PKG_VERSION:=1.8.8 -PKG_RELEASE:=3 +PKG_RELEASE:=4 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken diff --git a/net/banip/files/banip-functions.sh b/net/banip/files/banip-functions.sh index 1b45a06d5b..fe8ccf3a7b 100644 --- a/net/banip/files/banip-functions.sh +++ b/net/banip/files/banip-functions.sh @@ -172,7 +172,7 @@ f_mkdir() { if [ ! -d "${dir}" ]; then "${ban_rmcmd}" -f "${dir}" mkdir -p "${dir}" - f_log "debug" "f_mkdir ::: directory: ${dir}" + f_log "debug" "f_mkdir ::: directory: ${dir}" fi } @@ -347,7 +347,7 @@ f_conf() { # IPv4/IPv6 validation # f_chkip() { - local ipv type prefix separator col1 col2 + local ipv type prefix separator col1 col2 feed="${feed}" ipv="${1}" type="${2}" @@ -739,7 +739,7 @@ f_getelements() { # handle etag http header # f_etag() { - local http_head http_code etag_id etag_cnt out_rc="4" feed="${1}" feed_url="${2}" feed_suffix="${3}" feed_cnt="${4:-"1"}" + local http_head http_code etag_id etag_cnt etag_match result out_rc="4" feed="${1}" feed_url="${2}" feed_suffix="${3}" feed_cnt="${4:-"1"}" if [ -n "${ban_etagparm}" ]; then @@ -750,7 +750,7 @@ f_etag() { # fetch http headers and extract http code and etag/last-modified header # http_head="$("${ban_fetchcmd}" ${ban_etagparm} "${feed_url}" 2>&1)" - http_code="$(printf '%s' "${http_head}" | "${ban_awkcmd}" 'tolower($0)~/^http\/[0123\.]+ /{printf "%s",$2}')" + http_code="$(printf '%s' "${http_head}" | "${ban_awkcmd}" 'tolower($0)~/^[[:space:]]*http\/[0123\.]+ /{printf "%s",$2}')" etag_id="$(printf '%s' "${http_head}" | "${ban_awkcmd}" 'tolower($0)~/^[[:space:]]*etag: /{gsub("\"","");printf "%s",$2}')" # if etag header is not present, try to use last-modified header as fallback for change detection @@ -766,16 +766,19 @@ f_etag() { # compare http code and etag id with stored values, update etag file and return code accordingly # - etag_cnt="$("${ban_awkcmd}" -v f="${feed}" '$1 == f { n++ } END { print n+0 }' "${ban_backupdir}/banIP.etag")" - if [ "${http_code}" = "200" ] && [ "${etag_cnt}" = "${feed_cnt}" ] && [ -n "${etag_id}" ] && - "${ban_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" -v e="${etag_id}" ' - BEGIN { rc = 1; p = f " " s } - index($0, p) == 1 { - rest = substr($0, length(p) + 1) - sub(/^[[:space:]]+/, "", rest) - if (rest == e) { rc = 0; exit } - } - END { exit rc }' "${ban_backupdir}/banIP.etag"; then + result="$("${ban_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" -v e="${etag_id}" ' + BEGIN { p = f " " s; pl = length(p); m = 1 } + $1 == f { n++ } + index($0, p) == 1 { + rest = substr($0, pl + 1) + sub(/^[[:space:]]+/, "", rest) + if (rest == e) m = 0 + } + END { print n+0, m }' "${ban_backupdir}/banIP.etag")" + etag_cnt="${result% *}" + etag_match="${result#* }" + + if [ "${http_code}" = "200" ] && [ "${etag_cnt}" = "${feed_cnt}" ] && [ -n "${etag_id}" ] && [ "${etag_match}" = "0" ]; then out_rc="0" elif [ -n "${etag_id}" ]; then @@ -788,8 +791,7 @@ f_etag() { else "${ban_awkcmd}" -v f="${feed}" -v s="${feed_suffix}" ' BEGIN { p = f " " s } - index($0, p) != 1' \ - "${ban_backupdir}/banIP.etag" >"${ban_backupdir}/banIP.etag.new" + index($0, p) != 1' "${ban_backupdir}/banIP.etag" >"${ban_backupdir}/banIP.etag.new" fi "${ban_mvcmd}" -f "${ban_backupdir}/banIP.etag.new" "${ban_backupdir}/banIP.etag" printf '%s\t%s\n' "${feed} ${feed_suffix}" "${etag_id}" >>"${ban_backupdir}/banIP.etag" @@ -1485,7 +1487,6 @@ f_down() { fi fi fi - [ "${feed_rc}" != "0" ] && f_log "info" "download for feed '${feed}' failed, rc: ${feed_rc:-"-"}" # backup/restore # @@ -1496,6 +1497,7 @@ f_down() { f_restore "${feed}" "${feed_url}" "${tmp_load}" "${feed_rc}" feed_rc="${?}" fi + [ "${feed_rc}" != "0" ] && f_log "info" "processing for feed '${feed}' failed, rc: ${feed_rc:-"-"}" # final file & Set preparation for regular downloads # @@ -1803,7 +1805,6 @@ f_genstatus() { # : >"${ban_rtfile}" json_init - json_load_file "${ban_rtfile}" >/dev/null 2>&1 json_add_string "status" "${status}" json_add_string "frontend_ver" "${ban_fver}" json_add_string "backend_ver" "${ban_bver}" @@ -1887,34 +1888,76 @@ f_getstatus() { # domain lookup # f_lookup() { - local cnt list timestamp domain lookup ip elementsv4 elementsv6 start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}" + local cnt list domain lookup ip dom ts proto elementsv4 elementsv6 start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}" + local record_file tmp_dir target_file auto_flag + # measure runtime of lookup function for performance insights + # read -r start_time _ <"/proc/uptime" start_time="${start_time%%.*}" + + # prepare list of domains to lookup, target file for auto-adding new entries and auto-add flag based on feed type + # if [ "${feed}" = "allowlist" ]; then list="$("${ban_awkcmd}" '{gsub(/\r/,"")}/^([[:alnum:]_-]{1,63}\.)+[[:alpha:]]+([[:space:]]|$)/{printf "%s ",tolower($1)}' "${ban_allowlist}" 2>>"${ban_errorlog}")" + target_file="${ban_allowlist}" + auto_flag="${ban_autoallowlist}" elif [ "${feed}" = "blocklist" ]; then list="$("${ban_awkcmd}" '{gsub(/\r/,"")}/^([[:alnum:]_-]{1,63}\.)+[[:alpha:]]+([[:space:]]|$)/{printf "%s ",tolower($1)}' "${ban_blocklist}" 2>>"${ban_errorlog}")" + target_file="${ban_blocklist}" + auto_flag="${ban_autoblocklist}" fi + # prepare temporary directory for parallel lookups + # + tmp_dir="${ban_tmpfile}.lookup.${feed}" + f_mkdir "${tmp_dir}" + + # parallel DNS lookups: one record file per domain, network-bound work runs concurrently + # + cnt="1" for domain in ${list}; do - lookup="$("${ban_lookupcmd}" "${domain}" ${ban_resolver} 2>>"${ban_errorlog}" | "${ban_awkcmd}" '/^Address[ 0-9]*: /{if(!seen[$NF]++)printf "%s ",$NF}' 2>>"${ban_errorlog}")" - [ -n "${lookup}" ] && timestamp="$(date "+%Y-%m-%d %H:%M:%S")" - for ip in ${lookup}; do - if [ "${ip%%.*}" = "127" ] || [ "${ip%%.*}" = "0" ] || [ -z "${ip%%::*}" ]; then - continue - else - [ "${ip##*:}" = "${ip}" ] && elementsv4="${elementsv4} ${ip}," || elementsv6="${elementsv6} ${ip}," - if [ "${feed}" = "allowlist" ] && [ "${ban_autoallowlist}" = "1" ] && ! "${ban_grepcmd}" -q "^${ip}[[:space:]]*#" "${ban_allowlist}"; then - printf "%-45s%s\n" "${ip}" "# '${domain}' added on ${timestamp}" >>"${ban_allowlist}" - elif [ "${feed}" = "blocklist" ] && [ "${ban_autoblocklist}" = "1" ] && ! "${ban_grepcmd}" -q "^${ip}[[:space:]]*#" "${ban_blocklist}"; then - printf "%-45s%s\n" "${ip}" "# '${domain}' added on ${timestamp}" >>"${ban_blocklist}" + ( + lookup="$("${ban_lookupcmd}" "${domain}" ${ban_resolver} 2>>"${ban_errorlog}" | "${ban_awkcmd}" '/^Address[ 0-9]*: /{if(!seen[$NF]++)printf "%s ",$NF}' 2>>"${ban_errorlog}")" + [ -z "${lookup}" ] && exit 0 + ts="$(date "+%Y-%m-%d %H:%M:%S")" + for ip in ${lookup}; do + if [ "${ip%%.*}" = "127" ] || [ "${ip%%.*}" = "0" ] || [ -z "${ip%%::*}" ]; then + continue fi - cnt_ip="$((cnt_ip + 1))" - fi - done - cnt_domain="$((cnt_domain + 1))" + if [ "${ip##*:}" = "${ip}" ]; then + printf 'v4 %s %s %s\n' "${ip}" "${domain}" "${ts}" + else + printf 'v6 %s %s %s\n' "${ip}" "${domain}" "${ts}" + fi + done >"${tmp_dir}/${cnt}" + ) & + [ "${cnt}" -gt "${ban_cores}" ] && wait -n + cnt_domain="${cnt}" + cnt="$((cnt + 1))" done + wait + + # collect results: aggregate IPs, persist new entries serially (no append race) + # + for record_file in "${tmp_dir}"/*; do + [ -s "${record_file}" ] || continue + while read -r proto ip dom ts; do + cnt_ip="$((cnt_ip + 1))" + if [ "${proto}" = "v4" ]; then + elementsv4="${elementsv4} ${ip}," + else + elementsv6="${elementsv6} ${ip}," + fi + if [ "${auto_flag}" = "1" ] && ! "${ban_grepcmd}" -q "^${ip}[[:space:]]*#" "${target_file}"; then + printf "%-45s%s\n" "${ip}" "# '${dom}' added on ${ts}" >>"${target_file}" + fi + done <"${record_file}" + done + f_rmdir "${tmp_dir}" + + # add resolved IPs to nftables Sets + # if [ -n "${elementsv4}" ]; then if ! "${ban_nftcmd}" add element inet banIP "${feed}.v4" { ${elementsv4} } 2>>"${ban_errorlog}"; then f_log "info" "can't add lookup file to nfset '${feed}.v4'" @@ -1925,6 +1968,9 @@ f_lookup() { f_log "info" "can't add lookup file to nfset '${feed}.v6'" fi fi + + # measure end time and log performance insights + # read -r end_time _ <"/proc/uptime" end_time="${end_time%%.*}" duration="$(((end_time - start_time) / 60))m $(((end_time - start_time) % 60))s" @@ -1937,7 +1983,7 @@ f_lookup() { f_report() { local report_jsn report_txt tmp_val table_json item sep table_sets set_cnt set_inbound set_outbound set_cntinbound set_cntoutbound set_proto set_dport set_details local cnt ip expr detail jsnval timestamp autoadd_allow autoadd_block sum_sets sum_setinbound sum_setoutbound sum_cntelements sum_cntinbound sum_cntoutbound quantity - local chunk jsn map_jsn chain set_elements set_json sum_setelements sum_synflood sum_udpflood sum_icmpflood sum_ctinvalid sum_tcpinvalid sum_setports sum_bcp38 output="${1}" + local chunk jsn table_jsn set_jsn map_jsn chain set_elements sum_setelements sum_synflood sum_udpflood sum_icmpflood sum_ctinvalid sum_tcpinvalid sum_setports sum_bcp38 output="${1}" f_conf f_mkdir "${ban_reportdir}" @@ -1950,8 +1996,10 @@ f_report() { # json output preparation # : >"${report_txt}" >"${report_jsn}" >"${map_jsn}" - table_json="$("${ban_nftcmd}" -tj list table inet banIP 2>>"${ban_errorlog}")" - table_sets="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -qe '@.nftables[@.set.family="inet"].set.name')" + [ "${output}" = "gen" ] && printf '%s\n' "0" >"${ban_rundir}/banIP.report" + table_jsn="${ban_rundir}/report.table.jsn" + "${ban_nftcmd}" -tj list table inet banIP 2>>"${ban_errorlog}" >"${table_jsn}" + table_sets="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.set.family="inet"].set.name')" sum_sets="0" sum_cntelements="0" sum_setinbound="0" @@ -1960,19 +2008,20 @@ f_report() { sum_cntoutbound="0" sum_setports="0" sum_setelements="0" - sum_synflood="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt_synflood"].*.packets')" - sum_udpflood="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt_udpflood"].*.packets')" - sum_icmpflood="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt_icmpflood"].*.packets')" - sum_ctinvalid="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt_ctinvalid"].*.packets')" - sum_tcpinvalid="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt_tcpinvalid"].*.packets')" - sum_bcp38="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -qe '@.nftables[@.counter.name="cnt_bcp38"].*.packets')" + sum_synflood="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_synflood"].*.packets')" + sum_udpflood="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_udpflood"].*.packets')" + sum_icmpflood="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_icmpflood"].*.packets')" + sum_ctinvalid="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_ctinvalid"].*.packets')" + sum_tcpinvalid="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_tcpinvalid"].*.packets')" + sum_bcp38="$("${ban_jsoncmd}" -i "${table_jsn}" -qe '@.nftables[@.counter.name="cnt_bcp38"].*.packets')" timestamp="$(date "+%Y-%m-%d %H:%M:%S")" cnt="1" for item in ${table_sets}; do ( - set_json="$("${ban_nftcmd}" -j list set inet banIP "${item}" 2>>"${ban_errorlog}")" - set_cnt="$(printf '%s' "${set_json}" | "${ban_jsoncmd}" -qe '@.nftables[*].set.elem[*]' | "${ban_wccmd}" -l 2>>"${ban_errorlog}")" + set_jsn="${ban_rundir}/report.set.jsn.${item}" + "${ban_nftcmd}" -j list set inet banIP "${item}" 2>>"${ban_errorlog}" >"${set_jsn}" + set_cnt="$("${ban_jsoncmd}" -i "${set_jsn}" -qe '@.nftables[*].set.elem[*]' | "${ban_wccmd}" -l 2>>"${ban_errorlog}")" set_cntinbound="" set_cntoutbound="" set_inbound="" @@ -1983,17 +2032,18 @@ f_report() { for chain in _inbound _outbound; do for expr in 0 1 2; do if [ "${chain}" = "_inbound" ] && [ -z "${set_cntinbound}" ]; then - set_cntinbound="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" + set_cntinbound="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" elif [ "${chain}" = "_outbound" ] && [ -z "${set_cntoutbound}" ]; then - set_cntoutbound="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" + set_cntoutbound="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[${expr}].match.right=\"@${item}\"].expr[*].counter.packets")" fi - [ -z "${set_proto}" ] && set_proto="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[0].match.right.set")" - [ -z "${set_proto}" ] && set_proto="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.left.payload.protocol")" - [ -z "${set_dport}" ] && set_dport="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[1].match.right.set")" - [ -z "${set_dport}" ] && set_dport="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[1].match.right")" - [ -z "${set_dport}" ] && set_dport="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.right.set")" - [ -z "${set_dport}" ] && set_dport="$(printf '%s' "${table_json}" | "${ban_jsoncmd}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.right")" + [ -z "${set_proto}" ] && set_proto="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[0].match.right.set")" + [ -z "${set_proto}" ] && set_proto="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.left.payload.protocol")" + [ -z "${set_dport}" ] && set_dport="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[1].match.right.set")" + [ -z "${set_dport}" ] && set_dport="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[2].match.right=\"@${item}\"].expr[1].match.right")" + [ -z "${set_dport}" ] && set_dport="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.right.set")" + [ -z "${set_dport}" ] && set_dport="$("${ban_jsoncmd}" -i "${table_jsn}" -ql1 -e "@.nftables[@.rule.chain=\"${chain}\"][@.expr[1].match.right=\"@${item}\"].expr[0].match.right")" done + [ -n "${set_cntinbound}" ] && [ -n "${set_cntoutbound}" ] && [ -n "${set_proto}" ] && [ -n "${set_dport}" ] && break done if [ -n "${set_proto}" ] && [ -n "${set_dport}" ]; then set_proto="${set_proto//[\{\}\":]/}" @@ -2005,7 +2055,7 @@ f_report() { set_dport="${set_proto}: $(f_trim "${set_dport}")" fi if [ "${ban_nftcount}" = "1" ]; then - set_elements="$(printf '%s' "${set_json}" | "${ban_jsoncmd}" -l50 -qe '@.nftables[*].set.elem[*][@.counter.packets>0].val' | + set_elements="$("${ban_jsoncmd}" -i "${set_jsn}" -l50 -qe '@.nftables[*].set.elem[*][@.counter.packets>0].val' | "${ban_awkcmd}" -F '[ ,]' '{ORS=" ";if($2=="\"range\":"||$2=="\"concat\":")printf"%s, ",$4;else if($2=="\"prefix\":")printf"%s, ",$5;else printf"\"%s\", ",$1}')" fi if [ -n "${set_cntinbound}" ]; then @@ -2028,11 +2078,13 @@ f_report() { \"port\": \"${set_dport:-"-"}\", \ \"set_elements\": [ ${set_elements%%??} ] \ }" >"${report_jsn}.${item}" + "${ban_rmcmd}" -f "${set_jsn}" ) & [ "${cnt}" -gt "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done wait + "${ban_rmcmd}" -f "${table_jsn}" # assemble JSON from per-set fragments # diff --git a/net/banip/files/banip-service.sh b/net/banip/files/banip-service.sh index 1d158e338f..9f1a56cc87 100755 --- a/net/banip/files/banip-service.sh +++ b/net/banip/files/banip-service.sh @@ -153,7 +153,6 @@ for feed in allowlist ${ban_feed} blocklist; do done wait f_rmset -f_rmdir "${ban_tmpdir}" f_genstatus "active" # start domain lookup @@ -176,6 +175,7 @@ f_log "info" "finish banIP processing" f_mail fi json_cleanup + f_rmdir "${ban_tmpdir}" rm -rf "${ban_lock}" ) &