Linux firewalling Threat intelligence script

This is a threat intelligence script to use on a linux based network aggregator. It is based on this one.

First install the prerequisites.

sudo apt-get install ipset iprange

Place the file updatethreatblock in the directory /usr/local/sbin.

#!/usr/bin/env bash
#
# usage updatethreatblock.sh <configuration file>
# eg: updatethreatblock.sh /etc/ipset-threatblock/ipset-threatblock.conf
#
function exists() { command -v "$1" >/dev/null 2>&1 ; }

if [[ -z "$1" ]]; then
  echo "Error: please specify a configuration file, e.g. $0 /etc/ipset-threatblock/ipset-threatblock.conf"
  exit 1
fi

# shellcheck source=ipset-threatblock.conf
if ! source "$1"; then
  echo "Error: can't load configuration file $1"
  exit 1
fi

if ! exists curl && exists egrep && exists grep && exists ipset && exists iptables && exists sed && exists sort && exists wc ; then
  echo >&2 "Error: searching PATH fails to find executables among: curl egrep grep ipset iptables sed sort wc"
  exit 1
fi

DO_OPTIMIZE_CIDR=no
if exists iprange && [[ ${OPTIMIZE_CIDR:-yes} != no ]]; then
  DO_OPTIMIZE_CIDR=yes
fi

if [[ ! -d $(dirname "$IP_BLACKLIST") || ! -d $(dirname "$IP_BLACKLIST_RESTORE") ]]; then
  echo >&2 "Error: missing directory(s): $(dirname "$IP_BLACKLIST" "$IP_BLACKLIST_RESTORE"|sort -u)"
  exit 1
fi

if [ -f "$IP_BLACKLIST_EXCEPTIONS" ]; then
EXCEPTIONS_TMP=$(mktemp)
for exception in $(sed -r -e 's/\s*#.*$//;/^$/d;/^(0.0.0.0|10.|127.|172.1[6-9].|172.2[0-9].|172.3[0-1].|192.168.|22[4-9].|23[0-9].)/d' "$IP_BLACKLIST_EXCEPTIONS")
do
exception_array+=( "$exception" )
echo $exception >> $EXCEPTIONS_TMP
done
fi

# create the ipset if needed (or abort if does not exists and FORCE=no)
if ! ipset list -n|command grep -q "$IPSET_BLACKLIST_NAME"; then
  if [[ ${FORCE:-no} != yes ]]; then
    echo >&2 "Error: ipset does not exist yet, add it using:"
    echo >&2 "# ipset create $IPSET_BLACKLIST_NAME -exist hash:net family inet hashsize ${HASHSIZE:-16384} maxelem ${MAXELEM:-65536}"
    exit 1
  fi
  if ! ipset create "$IPSET_BLACKLIST_NAME" -exist hash:net family inet hashsize "${HASHSIZE:-16384}" maxelem "${MAXELEM:-65536}"; then
    echo >&2 "Error: while creating the initial ipset"
    exit 1
  fi
fi

# create the iptables binding if needed (or abort if does not exists and FORCE=no)
if ! iptables -nvL INPUT|command grep -q "match-set $IPSET_BLACKLIST_NAME"; then
  # we may also have assumed that INPUT rule n°1 is about packets statistics (traffic monitoring)
  if [[ ${FORCE:-no} != yes ]]; then
    echo >&2 "Error: iptables does not have the needed ipset INPUT rule, add it using:"
    echo >&2 "# iptables -I INPUT ${IPTABLES_IPSET_RULE_NUMBER:-1} -m set --match-set $IPSET_BLACKLIST_NAME src -j DROP"
    exit 1
  fi
  if ! iptables -I INPUT "${IPTABLES_IPSET_RULE_NUMBER:-1}" -m set --match-set "$IPSET_BLACKLIST_NAME" src -j DROP; then
    echo >&2 "Error: while adding the --match-set ipset rule to iptables"
    exit 1
  fi
fi

IP_BLACKLIST_TMP=$(mktemp)
for i in "${BLACKLISTS[@]}"
do
  IP_TMP=$(mktemp)
  (( HTTP_RC=$(curl -L -A "blacklist-update/script/github" --connect-timeout 10 --max-time 10 -o "$IP_TMP" -s -w "%{http_code}" "$i") ))
  if (( HTTP_RC == 200 || HTTP_RC == 302 || HTTP_RC == 0 )); then # "0" because file:/// returns 000
    command grep -Po '^(?:\d{1,3}\.){3}\d{1,3}(?:/\d{1,2})?' "$IP_TMP" | sed -r 's/^0*([0-9]+)\.0*([0-9]+)\.0*([0-9]+)\.0*([0-9]+)$/\1.\2.\3.\4/' >> "$IP_BLACKLIST_TMP"
    [[ ${VERBOSE:-yes} == yes ]] && echo -n "."
  elif (( HTTP_RC == 503 )); then
    echo -e "\\nUnavailable (${HTTP_RC}): $i"
  else
    echo >&2 -e "\\nWarning: curl returned HTTP response code $HTTP_RC for URL $i"
  fi
  rm -f "$IP_TMP"
done

# sort -nu does not work as expected
sed -r -e '/^(0\.0\.0\.0|10\.|127\.|172\.1[6-9]\.|172\.2[0-9]\.|172\.3[0-1]\.|192\.168\.|22[4-9]\.|23[0-9]\.)/d' "$IP_BLACKLIST_TMP"|sort -n|sort -mu >| "$IP_BLACKLIST"
if [[ ${DO_OPTIMIZE_CIDR} == yes ]]; then
  if [[ ${VERBOSE:-no} == yes ]]; then
    echo -e "\\nAddresses before CIDR optimization: $(wc -l "$IP_BLACKLIST" | cut -d' ' -f1)"
  fi
  < "$IP_BLACKLIST" iprange --optimize - > "$IP_BLACKLIST_TMP" 2>/dev/null
  if [[ ${#exception_array[@]} > 0 ]]; then
    echo "Allowing for ${#exception_array[@]} exclusions from blacklist"
    echo "Addresses before removing exclusions: $(wc -l "$IP_BLACKLIST_TMP" | cut -d' ' -f1)"
    IP_BLACKLIST_WITH_EXCEPT_TMP=$(mktemp)
    iprange "$IP_BLACKLIST_TMP" --except "$EXCEPTIONS_TMP" > "$IP_BLACKLIST_WITH_EXCEPT_TMP" 2>/dev/null
    cp "$IP_BLACKLIST_WITH_EXCEPT_TMP" "$IP_BLACKLIST_TMP"
  fi
  if [[ ${VERBOSE:-no} == yes ]]; then
    echo "Addresses after CIDR optimization:  $(wc -l "$IP_BLACKLIST_TMP" | cut -d' ' -f1)"
  fi
  cp "$IP_BLACKLIST_TMP" "$IP_BLACKLIST"
fi

rm -f "$IP_BLACKLIST_TMP"

# family = inet for IPv4 only
cat >| "$IP_BLACKLIST_RESTORE" <<EOF
create $IPSET_TMP_BLACKLIST_NAME -exist hash:net family inet hashsize ${HASHSIZE:-16384} maxelem ${MAXELEM:-65536}
create $IPSET_BLACKLIST_NAME -exist hash:net family inet hashsize ${HASHSIZE:-16384} maxelem ${MAXELEM:-65536}
EOF

# can be IPv4 including netmask notation
# IPv6 ? -e "s/^([0-9a-f:./]+).*/add $IPSET_TMP_BLACKLIST_NAME \1/p" \ IPv6
sed -rn -e '/^#|^$/d' \
  -e "s/^([0-9./]+).*/add $IPSET_TMP_BLACKLIST_NAME \\1/p" "$IP_BLACKLIST" >> "$IP_BLACKLIST_RESTORE"

cat >> "$IP_BLACKLIST_RESTORE" <<EOF
swap $IPSET_BLACKLIST_NAME $IPSET_TMP_BLACKLIST_NAME
destroy $IPSET_TMP_BLACKLIST_NAME
EOF

ipset -file  "$IP_BLACKLIST_RESTORE" restore

if [[ ${VERBOSE:-no} == yes ]]; then
  echo
  echo "Threatblock addresses found: $(wc -l "$IP_BLACKLIST" | cut -d' ' -f1)"
fi

Make the script executable: sudo chmod +x /usr/local/sbin/updatethreatblock.sh

Create a directory for the config files: sudo mkdir /etc/ipset-threatblock

Create a direct for the log files: /var/log/threatblock

Create a configuration file ipset-threatblock.conf in /etc/ipset-threatblock

IPSET_BLACKLIST_NAME=threatblock # change it if it collides with a pre-existing ipset list
IPSET_TMP_BLACKLIST_NAME=${IPSET_BLACKLIST_NAME}-tmp

# ensure the directory for IP_BLACKLIST/IP_BLACKLIST_RESTORE exists (it won't be created automatically)
IP_BLACKLIST_RESTORE=/etc/ipset-threatblock/ip-threatblock.restore
IP_BLACKLIST=/etc/ipset-threatblock/ip-threatblock.list

IP_BLACKLIST_EXCEPTIONS=/etc/ipset-threatblock/threatblock.exceptions

VERBOSE=yes # probably set to "no" for cron jobs, default to yes
FORCE=yes # will create the ipset-iptable binding if it does not already exist
let IPTABLES_IPSET_RULE_NUMBER=1 # if FORCE is yes, the number at which place insert the ipset-match rule (default to 1)

# Sample (!) list of URLs for IP blacklists. Currently, only IPv4 is supported in this script, everything else will be filtered.
BLACKLISTS=(
    "file:///etc/ipset-blacklist/ip-threatblock-custom.list" # optional, for your personal nemeses (no typo, plural)
    "https://www.projecthoneypot.org/list_of_ips.php?t=d&rss=1" # Project Honey Pot Directory of Dictionary Attacker IPs
    "https://check.torproject.org/cgi-bin/TorBulkExitList.py?ip=1.1.1.1"  # TOR Exit Nodes
    "http://danger.rulez.sk/projects/bruteforceblocker/blist.php" # BruteForceBlocker IP List
    "https://www.spamhaus.org/drop/drop.lasso" # Spamhaus Don't Route Or Peer List (DROP)
    "https://cinsscore.com/list/ci-badguys.txt" # C.I. Army Malicious IP List
    "https://lists.blocklist.de/lists/all.txt" # blocklist.de attackers
    "https://blocklist.greensnow.co/greensnow.txt" # GreenSnow
    "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset" # Firehol Level 1
    "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/stopforumspam_7d.ipset" # Stopforumspam via Firehol
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_shadowserver.txt" # Shadow server
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_shodan.txt" # Shodan
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_sogou.txt" # Sogou search engine
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_evil.txt" # Hostile IPs
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_internet_cens.txt" # Internet census
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_diverseenvironment.txt" # Diverse environment
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_netsysres.txt" # Net Systems Research
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_rwth-aachen.txt" # AAChen
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_onyphe.txt" # onephe
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_stretchoid.txt" # Strechoid
    "https://gitlab.com/ohisee/block-shodan-stretchoid-census/raw/master/pf_table_openportsstats.txt" # OpenPortStats
    # "http://ipverse.net/ipblocks/data/countries/xx.zone" # Ban an entire country, see http://ipverse.net/ipblocks/data/countries/
)
MAXELEM=131072

Use a threatblock.exceptions file in /etc/ipset-threatblock to overwrite blocks IPs.

Use a ip-threatblock-custom.list file in /etc/ipset-threatblock to add custom IPs for blocking.

Create a script startubl.sh in /usr/local/sbin to use in crontab

#!/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
cd /usr/local/sbin
date > /var/log/threatblock/tb-`date +%d%H`.run
/usr/local/sbin/updatethreatblock.sh /etc/ipset-threatblock/ipset-threatblock.conf
date >> /var/log/threatblock/tb-`date +%d%H`.run

Create a script rebootubl.sh in /usr/local/sbin

#!/bin/bash
# Threat intelligence
/sbin/ipset restore < /etc/ipset-threatblock/ip-threatblock.restore
/sbin/iptables -I INPUT   1 -i enp4s0f1 -m set --match-set threatblock src -j DROP -m comment --comment "threat intel"
/sbin/iptables -I FORWARD 1 -i enp4s0f1 -m set --match-set threatblock src -j DROP -m comment --comment "threat intel"
/sbin/iptables -I FORWARD 1 -o enp4s0f1 -m set --match-set threatblock src -j DROP -m comment --comment "threat intel"

Update crontab: sudo crontab -e

#
# m h  dom mon dow   command
@reboot         sleep 180 && /usr/local/sbin/rebootubl.sh
@daily          /usr/local/sbin/startubl.sh

FROM: https://www.linkedin.com/pulse/threat-intelligence-script-ronald-bartels?trk=articles_directory