Skip to content

N
nexdpi
Commit 2b7baae4 authored by root's avatar root
Browse files
Options

Add initial release

parent 3dfb2129
Hide whitespace changes
Inline Side-by-side
Showing with 460 additions and 0 deletions
dpi 0 → 100755
View file @ 2b7baae4
#!/usr/bin/python3
from setproctitle import setproctitle
from shell_cmd import sh
from nfstream import NFStreamer
import time
import sys
import json
deftimeout="3600"
setproctitle("dpi")
online_streamer = NFStreamer(source="br0", promiscuous_mode=False, splt_analysis=20, statistical_analysis=False)
templconf = """
{
"Cats":{
"Network":{
"nostart": [],
"noapps": [],
"knownapps": ["DHCP", "DHCPV6", "DNScrypt", "DoH_DoT", "DNS", "Ookla", "ICMP", "ICMPV6", "IGMP", "LLMNR", "MDNS", "DoH_DoT", "Ookla", "WSD"],
"ipset": "system_triplet",
"timeout":"3600"
},
"Game":{
"nostart": [],
"noapps": [],
"knownapps": ["Steam", "Xbox", "Playstation"],
"ipset": "streaming_triplet",
"timeout":"3600"
},
"Template-GroupName":{
"nostart": ["DNS", "ICMP"],
"noapps": [],
"knownapps": [],
"ipset": "name_ipset",
"timeout":"3600"
}
},
"Apps":{
"TikTok":{
"ipset": "kids_triplet",
"nostart": [],
"timeout": "3600",
"knowstarts":"TLS"
}
}
}
"""
try:
fconf = open("/etc/nexdpi/dpirules.conf", "r")
R=json.loads(fconf.read())
fconf.close()
except:
print(" Cant start without a rules file.\n\n")
print(" Please use the following example to create a JSON config file as /etc/nexdpi/dpirules.conf\n\n")
print(templconf)
sys.exit(0)
Cats = R['Cats']
Apps = R['Apps']
UnknownMatch=[]
class NexDPI():
fullname=False
isknown=False
def __init__(self):
print(time.asctime(), "NexDPI created")
def main(self):
print(time.asctime(), "NexDPI started")
for flow in online_streamer:
if self.fullname and not self.fullname in UnknownMatch and not self.isknown:
print(self.fullname)
UnknownMatch.append(self.fullname)
self.isknown = False
self.fullname=flow.application_name+" "+flow.application_category_name
triplet=str(flow.dst_ip)+","+str(flow.dst_port)+","+str(flow.src_ip)
cname = flow.application_category_name
aname = flow.application_name
sername = aname.split(".")[-1:][0]
ipv=flow.ip_version
if cname in list(Cats.keys()):
if ipv==6:
ipset_list = Cats[cname]['ipset']+"6"
else:
ipset_list = Cats[cname]['ipset']
if not aname.startswith(Cats[cname]['nostart']) and not sername in Cats[cname]['noapps']:
sh("ipset test "+ipset_list+" "+triplet+" >/dev/null 2>&1 || ipset add "+ipset_list+" "+triplet+" timeout "+Cats[cname]['timeout']+" > /dev/null 2>&1")
if sername in Cats[cname]['knownapps']:
self.isknown = True
continue
if sername in list(Apps.keys()):
appd=Apps[sername]
if ipv==6:
ipset_list = appd['ipset']+"6"
else:
ipset_list = appd['ipset']
if not aname.startswith(appd['nostart']):
sh("ipset test "+ipset_list+" "+triplet+" >/dev/null 2>&1 || ipset add "+ipset_list+" "+triplet+" timeout "+appd['timeout']+" > /dev/null 2>&1")
if aname.startswith(appd['knowstarts']):
self.isknown = True
continue
#print(flow.application_is_guessed)
#print(flow.src_ip)
#print(flow.src_port)
#print(flow.dst_ip)
#print(flow.dst_port)
#print(flow.requested_server_name)
#time.sleep(0.1)
if __name__ == "__main__":
import sys
ndpi=NexDPI()
ndpi.main()
shaping 0 → 100755
View file @ 2b7baae4
#!/bin/bash -x
declare -A SHAPEMAPDOWN
declare -A SHAPEMAPUP
declare -A IPSETS
UPDEV="eth0"
#DOWNDEV="ifb0"
#DOWNDEV="dummy1"
#DOWNDEV="eth1"
DOWNDEV="br0"
IP="/sbin/ip"
TC="/sbin/tc"
IPSET="/usr/sbin/ipset"
IPT="/usr/sbin/iptables"
IP6T="/usr/sbin/ip6tables"
FORCEIFB=false
DEFBAND="980"
DOUPLOAD=true
DODOWNLOAD=true
IPSET_DURATION=3600 # Seconds
IPSETS_NAMES="social kids system full streaming"
NAT=true
MARKSTART=5
for IPS in $IPSETS_NAMES ; do
IPSETS[$IPS]=$MARKSTART
((MARKSTART=MARKSTART+1))
done
IPTMARK="$IPT -A PREROUTING -t mangle"
IPTCHECK="$IPT -C PREROUTING -t mangle"
IPTDEL="$IPT -D PREROUTING -t mangle"
IP6TMARK="$IP6T -A PREROUTING -t mangle"
IP6TCHECK="$IP6T -C PREROUTING -t mangle"
IP6TDEL="$IP6T -D PREROUTING -t mangle"
iptmark() {
$IPTCHECK $* >/dev/null 2>&1 || $IPTMARK $*
}
iptdel() {
$IPTCHECK $* >/dev/null 2>&1 && $IPTDEL $* || true
}
ip6mark() {
$IP6TCHECK $* >/dev/null 2>&1 || $IP6TMARK $*
}
ip6del() {
$IP6TCHECK $* >/dev/null 2>&1 && $IP6TDEL $* || true
}
if [ "$DOWNDEV" == "ifb0" ] ; then
FORCEIFB=true
fi
if [ x"$(brctl show | grep $DOWNDEV)" != "" ] ; then
DOWNDEV=$(ls /sys/class/net/br0/brif/)
fi
# NOTE: Express in mbit, a bit less than the actual connections helps
# ours qdisc to be in charge when full.
# For 1Gbps for example uses 980
for DEV in $UPDEV ; do
SHAPEMAPUP[$DEV]=$DEFBAND
done
for DEV in $DOWNDEV ; do
SHAPEMAPDOWN[$DEV]=$DEFBAND
done
if [ x"$1" == x"off" ] ; then
for DEV in $DOWNDEV ; do
$TC qdisc del dev $DEV root > /dev/null 2>&1 || true
if $FORCEIFB ; then
$IP link set dev $DEV down
fi
done
for DEV in $UPDEV ; do
$TC qdisc del dev $DEV root > /dev/null 2>&1 || true
$TC qdisc del dev $DEV ingress > /dev/null 2>&1 || true
done
exit 0
else
for IPS in $IPSETS_NAMES ; do
$IPSET create ${IPS}_triplet hash:ip,port,ip family inet timeout $IPSET_DURATION > /dev/null 2>&1 || true
$IPSET create ${IPS}_ip hash:ip family inet > /dev/null 2>&1 || true
$IPSET create ${IPS}_extip hash:ip family inet > /dev/null 2>&1 || true
$IPSET create ${IPS}_triplet6 hash:ip,port,ip family inet6 timeout $IPSET_DURATION > /dev/null 2>&1 || true
$IPSET create ${IPS}_ip6 hash:ip family inet6 > /dev/null 2>&1 || true
$IPSET create ${IPS}_extip6 hash:ip family inet6 > /dev/null 2>&1 || true
done
fi
# Setting up virtual mirror interface for downloads
# XXX Works only with 1 ifb device!
#
# XXX ifb has AWFUL performances.
# Don't use it if you want to achieve anything more than 600Mbps.
if $FORCEIFB ; then
if [ "$DOWNDEV" == "ifb0" ] ; then
modprobe ifb
fi
$IP link set dev $DOWNDEV up
# Redirect incoming packet from WAN to the virtual interface
$TC qdisc add dev $UPDEV handle ffff: ingress
# Filter to make the packet going on virtual interface
$TC filter add dev $UPDEV protocol ip parent ffff: u32 match u32 0 0 action mirred egress redirect dev $DOWNDEV
fi
#
# Shaping scheme
#
# .-------------------------------------------------------------------------------------------------------.
# | |
# | HTB Base 1: |
# | .---------------------------------------------------------------------------------------------------. |
# | | | |
# | | Class 1:1 | |
# | | .------------------------------------------------------------. .---------------..---------------. | |
# | | | | | || | | |
# | | | Class 1:10 | | Class 1:20 || Class 1:30 | | |
# | | | .--------------------------------------------------------. | | .-----------. || .-----------. | | |
# | | | | | | | | | || | | | | |
# | | | | HTB 11: | | | | fq_codel | || | fq_codel | | | |
# | | | | .----------------..----------------..----------------. | | | | | || | | | | |
# | | | | | || || | | | | '-----------' || '-----------' | | |
# | | | | | Class 11:10 || Class 11:12 || Class 11:16 | | | '---------------''---------------' | |
# | | | | | .------------. || .------------. || .------------. | | | .---------------..---------------. | |
# | | | | | | | || | | || | | | | | | || | | |
# | | | | | | fq_codel | || | fq_codel | || | fq_codel | | | | | Class 1:40 || Class 1:50 | | |
# | | | | | | | || | | || | | | | | | .-----------. || .-----------. | | |
# | | | | | '------------' || '------------' || '------------' | | | | | | || | | | | |
# | | | | | || || | | | | | fq_codel | || | fq_codel | | | |
# | | | | '----------------''----------------''----------------' | | | | | || | | | | |
# | | | '--------------------------------------------------------' | | '-----------' || '-----------' | | |
# | | '------------------------------------------------------------' '---------------''---------------' | |
# | '---------------------------------------------------------------------------------------------------' |
# '-------------------------------------------------------------------------------------------------------'
#
# HTB Base 1: Main bucket, HTB qdisc on root, 950Mbit/1Gbit
# '-Class 1:1 Main container class
# |
# |- Class 1:10 Default class for catchall all packets not for other classes
# | '-HTB 11: qdisc for catchall class
# | |
# | |-Class 11:10 normal priority packets
# | | '- fq_codel
# | |
# | |-Class 11:12 high priority packets (also streaming)
# | | '- fq_codel
# | |
# | '-Class 11:16 low priority packets
# | '- fq_codel
# |
# |-Class 1:20: system/infrastructure
# | '- fq_codel
# |
# |-Class 1:30: Kiddos
# | '- fq_codel
# |
# |-Class 1:40: SocialNetworks and similar
# | '- fq_codel
# |
# '-Class 1:50: work
# '- fq_codel
#
#exit 0
ApplyShaping() {
SHAPEDEV=$1
local TCLASS="/sbin/tc class add dev $SHAPEDEV parent"
local TDISC="/sbin/tc qdisc add dev $SHAPEDEV parent"
local TFILTER="/sbin/tc filter add dev $SHAPEDEV parent"
REALRATE=$2
DIRECTION=$3
local REDUCERATE=$(($REALRATE/30))
local MAXREDUCEDRATE=$(($REALRATE-$REDUCERATE))
local HALFRATE=$(($REALRATE/2))
local MINRATE=$(($REALRATE/10))
local FAIRRATE=$(($REALRATE/3))
[ x"$DIRECTION" == x"up" ] && INTDIR="src" || INTDIR="dst"
[ x"$DIRECTION" == x"up" ] && OUTDIR="dst" || OUTDIR="src"
# Check if we have to reset the interface to default
/sbin/tc qdisc del dev $SHAPEDEV root > /dev/null 2>&1 || true
# Change root qdisc
/sbin/tc qdisc replace dev $SHAPEDEV root handle 1: htb default 10
# Create base class
$TCLASS 1: classid 1:1 htb rate ${REALRATE}mbit
# Container qdisk and class for cathall
$TCLASS 1:1 classid 1:10 htb rate ${MAXREDUCEDRATE}mbit ceil ${REALRATE}mbit prio 1
$TDISC 1:10 handle 11: htb default 10
$TCLASS 11: classid 11:1 htb rate ${MAXREDUCEDRATE}mbit ceil ${REALRATE}mbit
# Catchall normal priority
$TCLASS 11:1 classid 11:10 htb rate ${MAXREDUCEDRATE}mbit ceil ${REALRATE}mbit prio 2
$TDISC 11:10 fq_codel
# Catchall low priority
$TCLASS 11:1 classid 11:12 htb rate ${FAIRRATE}mbit ceil ${REALRATE}mbit prio 3
$TDISC 11:12 fq_codel
# Catchall high priority
$TCLASS 11:1 classid 11:16 htb rate ${MINRATE}mbit ceil ${FAIRRATE}mbit prio 1
$TDISC 11:16 fq_codel
# Classificy and filter based on priority in the packet
$TFILTER 11: handle 0x1337 flow map key priority baseclass 11:10
# System/infrastructure
$TCLASS 1:1 classid 1:20 htb rate ${MINRATE}mbit ceil ${REALRATE}mbit prio 2
$TDISC 1:20 fq_codel
# Kiddos
$TCLASS 1:1 classid 1:30 htb rate ${FAIRRATE}mbit ceil ${HALFRATE}mbit prio 2
$TDISC 1:30 fq_codel
# SocialNetworks
$TCLASS 1:1 classid 1:40 htb rate ${MINRATE}mbit ceil ${FAIRRATE}mbit prio 3
$TDISC 1:40 fq_codel
# Work / Full Speed
$TCLASS 1:1 classid 1:50 htb rate ${MAXREDUCEDRATE}mbit ceil ${REALRATE}mbit prio 1
$TDISC 1:50 fq_codel
$TFILTER 1: protocol ip prio 4 basic match "ipset(social_extip $OUTDIR)" flowid 1:40
$TFILTER 1: protocol ip prio 3 basic match "ipset(social_ip $INTDIR)" flowid 1:40
$TFILTER 1: protocol ip prio 2 basic match "ipset(social_triplet $OUTDIR,$OUTDIR,$INTDIR)" flowid 1:40
$TFILTER 1: protocol ip prio 1 handle ${IPSETS["social"]} fw flowid 1:40
$TFILTER 1: protocol ipv6 prio 8 basic match "ipset(social_extip6 $OUTDIR)" flowid 1:40
$TFILTER 1: protocol ipv6 prio 7 basic match "ipset(social_ip6 $INTDIR)" flowid 1:40
$TFILTER 1: protocol ipv6 prio 6 basic match "ipset(social_triplet6 $OUTDIR,$OUTDIR,$INTDIR)" flowid 1:40
$TFILTER 1: protocol ipv6 prio 5 handle ${IPSETS["social"]} fw flowid 1:40
$TFILTER 1: protocol ip prio 4 basic match "ipset(kids_extip $OUTDIR)" flowid 1:43
$TFILTER 1: protocol ip prio 3 basic match "ipset(kids_ip $INTDIR)" flowid 1:30
$TFILTER 1: protocol ip prio 2 basic match "ipset(kids_triplet $OUTDIR,$OUTDIR,$INTDIR)" flowid 1:30
$TFILTER 1: protocol ip prio 1 handle ${IPSETS["kids"]} fw flowid 1:30
$TFILTER 1: protocol ipv6 prio 8 basic match "ipset(kids_extip6 $OUTDIR)" flowid 1:43
$TFILTER 1: protocol ipv6 prio 7 basic match "ipset(kids_ip6 $INTDIR)" flowid 1:30
$TFILTER 1: protocol ipv6 prio 6 basic match "ipset(kids_triplet6 $OUTDIR,$OUTDIR,$INTDIR)" flowid 1:30
$TFILTER 1: protocol ipv6 prio 5 handle ${IPSETS["kids"]} fw flowid 1:30
$TFILTER 1: protocol ip prio 4 basic match "ipset(system_extip $OUTDIR)" flowid 1:20
$TFILTER 1: protocol ip prio 3 basic match "ipset(system_ip $INTDIR)" flowid 1:20
$TFILTER 1: protocol ip prio 2 basic match "ipset(system_triplet $OUTDIR,$OUTDIR,$INTDIR)" flowid 1:20
$TFILTER 1: protocol ip prio 1 handle ${IPSETS["system"]} fw flowid 1:20
$TFILTER 1: protocol ipv6 prio 8 basic match "ipset(system_extip6 $OUTDIR)" flowid 1:20
$TFILTER 1: protocol ipv6 prio 7 basic match "ipset(system_ip6 $INTDIR)" flowid 1:20
$TFILTER 1: protocol ipv6 prio 6 basic match "ipset(system_triplet6 $OUTDIR,$OUTDIR,$INTDIR)" flowid 1:20
$TFILTER 1: protocol ipv6 prio 5 handle ${IPSETS["system"]} fw flowid 1:20
$TFILTER 1: protocol ip prio 4 basic match "ipset(full_extip $OUTDIR)" flowid 1:50
$TFILTER 1: protocol ip prio 3 basic match "ipset(full_ip $INTDIR)" flowid 1:50
$TFILTER 1: protocol ip prio 2 basic match "ipset(full_triplet $OUTDIR,$OUTDIR,$INTDIR)" flowid 1:50
$TFILTER 1: protocol ip prio 1 handle ${IPSETS["full"]} fw flowid 1:50
$TFILTER 1: protocol ipv6 prio 8 basic match "ipset(full_extip6 $OUTDIR)" flowid 1:50
$TFILTER 1: protocol ipv6 prio 7 basic match "ipset(full_ip6 $INTDIR)" flowid 1:50
$TFILTER 1: protocol ipv6 prio 6 basic match "ipset(full_triplet6 $OUTDIR,$OUTDIR,$INTDIR)" flowid 1:50
$TFILTER 1: protocol ipv6 prio 5 handle ${IPSETS["full"]} fw flowid 1:50
$TFILTER 1: protocol ip prio 4 basic match "ipset(streaming_extip $OUTDIR)" flowid 11:12
$TFILTER 1: protocol ip prio 3 basic match "ipset(streaming_ip $INTDIR)" flowid 11:12
$TFILTER 1: protocol ip prio 2 basic match "ipset(streaming_triplet $OUTDIR,$OUTDIR,$INTDIR)" flowid 11:12
$TFILTER 1: protocol ip prio 1 handle ${IPSETS["streaming"]} fw flowid 11:12
$TFILTER 1: protocol ipv6 prio 8 basic match "ipset(streaming_extip6 $OUTDIR)" flowid 11:12
$TFILTER 1: protocol ipv6 prio 7 basic match "ipset(streaming_ip6 $INTDIR)" flowid 11:12
$TFILTER 1: protocol ipv6 prio 6 basic match "ipset(streaming_triplet6 $OUTDIR,$OUTDIR,$INTDIR)" flowid 11:12
$TFILTER 1: protocol ipv6 prio 5 handle ${IPSETS["streaming"]} fw flowid 11:12
if [ x"$DIRECTION" == x"up" ] ; then
if $NAT ; then
for IPS in $IPSETS_NAMES ; do
iptmark -m set --match-set ${IPS}_ip $INTDIR -j MARK --set-mark ${IPSETS[$IPS]}
iptmark -m set --match-set ${IPS}_triplet $OUTDIR,$OUTDIR,$INTDIR -j MARK --set-mark ${IPSETS[$IPS]}
iptmark -m set --match-set ${IPS}_extip $OUTDIR -j MARK --set-mark ${IPSETS[$IPS]}
ip6mark -m set --match-set ${IPS}_ip6 $INTDIR -j MARK --set-mark ${IPSETS[$IPS]}
ip6mark -m set --match-set ${IPS}_triplet6 $OUTDIR,$OUTDIR,$INTDIR -j MARK --set-mark ${IPSETS[$IPS]}
ip6mark -m set --match-set ${IPS}_extip6 $OUTDIR -j MARK --set-mark ${IPSETS[$IPS]}
done
fi
fi
}
if $DODOWNLOAD ; then
for SHAPEDEV in "${!SHAPEMAPDOWN[@]}" ; do
REALRATE=${SHAPEMAPDOWN[$SHAPEDEV]}
ApplyShaping $SHAPEDEV $REALRATE down
done
fi
if $DOUPLOAD ; then
for SHAPEDEV in "${!SHAPEMAPUP[@]}" ; do
REALRATE=${SHAPEMAPUP[$SHAPEDEV]}
ApplyShaping $SHAPEDEV $REALRATE up
done
fi
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment