- WIP:

- new json configuration: added multiple configs, ip groups/ip aliases and timetables
  - added multiple configurations:
    - each configuration has his own upstream, singlefilters, doublefilters, blackholeip hostsfile
    - cache DB is global to all configs
    - BL downloader and parser is optimized: each BL source is downloaded/parsed only once
- TODO:
  - implement configuration selection based on source IPs and timetables
  - unused code cleanup

00.database.go

@@ -7,12 +7,15 @@ import (
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
-//MyZabovKDB is the storage where we'll put domains to block
+//MyZabovKDB is the storage where we'll put domains to block (obsolete)
-var MyZabovKDB *leveldb.DB
+//var MyZabovKDB *leveldb.DB
 
-//MyZabovCDB is the storage where we'll put domains to cache
+//MyZabovCDB is the storage where we'll put domains to cache (global for all configs)
 var MyZabovCDB *leveldb.DB
 
+//MyZabovKDBs is the storage where we'll put domains to block (one for each config)
+var MyZabovKDBs map[string]*leveldb.DB
+
 func init() {
 
 	var err error
@@ -20,13 +23,13 @@ func init() {
 	os.RemoveAll("./db")
 
 	os.MkdirAll("./db", 0755)
-
+	/*
-	MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil)
+		MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil)
-	if err != nil {
+		if err != nil {
-		fmt.Println("Cannot create Killfile db: ", err.Error())
+			fmt.Println("Cannot create Killfile db: ", err.Error())
-	} else {
+		} else {
-		fmt.Println("Killfile DB created")
+			fmt.Println("Killfile DB created")
-	}
+		}*/
 
 	MyZabovCDB, err = leveldb.OpenFile("./db/cache", nil)
 	if err != nil {
@@ -35,4 +38,21 @@ func init() {
 		fmt.Println("Cache DB created")
 	}
 
+	MyZabovKDBs = map[string]*leveldb.DB{}
+}
+
+// ZabovCreateKDB creates Kill DBs
+func ZabovCreateKDB(conf string) {
+	var err error
+
+	dbname := "./db/killfile_" + conf
+	KDB, err := leveldb.OpenFile(dbname, nil)
+	if err != nil {
+		fmt.Println("Cannot create Killfile db: ", err.Error())
+	} else {
+		fmt.Println("Killfile DB created")
+	}
+
+	MyZabovKDBs[conf] = KDB
+
 }

01.conf.go

@@ -6,29 +6,16 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 
 	"github.com/miekg/dns"
 )
 
+type stringarray []string
+type urlsMap map[string]stringarray
+
 func init() {
-
+	var MyConfRaw interface{}
-	//ZabovConf describes the Json we use for configuration
-	type ZabovConf struct {
-		Zabov struct {
-			Port          string `json:"port"`
-			Proto         string `json:"proto"`
-			Ipaddr        string `json:"ipaddr"`
-			Upstream      string `json:"upstream"`
-			Cachettl      int    `json:"cachettl"`
-			Killfilettl   int    `json:"killfilettl"`
-			Singlefilters string `json:"singlefilters"`
-			Doublefilters string `json:"doublefilters"`
-			Blackholeip   string `json:"blackholeip"`
-			Hostsfile     string `json:"hostsfile"`
-		} `json:"zabov"`
-	}
-
-	var MyConf ZabovConf
 
 	file, err := ioutil.ReadFile("config.json")
 
@@ -37,26 +24,35 @@ func init() {
 		os.Exit(1)
 	}
 
-	err = json.Unmarshal([]byte(file), &MyConf)
+	err = json.Unmarshal([]byte(file), &MyConfRaw)
 
 	if err != nil {
-		log.Println("Cannot marshal json: ", err.Error())
+		log.Println("Cannot unmarshal json: ", err.Error())
 		os.Exit(1)
 	}
 
 	// now we read configuration file
 	fmt.Println("Reading configuration file...")
 
-	ZabovPort := MyConf.Zabov.Port
+	MyConf := MyConfRaw.(map[string]interface{})
-	ZabovType := MyConf.Zabov.Proto
+
-	ZabovAddr := MyConf.Zabov.Ipaddr
+	zabov := MyConf["zabov"].(map[string]interface{})
-	ZabovUpDNS = MyConf.Zabov.Upstream
+
-	ZabovSingleBL = MyConf.Zabov.Singlefilters
+	ZabovPort := zabov["port"].(string)
-	ZabovDoubleBL = MyConf.Zabov.Doublefilters
+	ZabovType := zabov["proto"].(string)
-	ZabovAddBL = MyConf.Zabov.Blackholeip
+	ZabovAddr := zabov["ipaddr"].(string)
-	ZabovCacheTTL = MyConf.Zabov.Cachettl
+	ZabovCacheTTL = int(zabov["cachettl"].(float64))
-	ZabovKillTTL = MyConf.Zabov.Killfilettl
+	ZabovKillTTL = int(zabov["killfilettl"].(float64))
-	ZabovHostsFile = MyConf.Zabov.Hostsfile
+
+	configs := MyConf["configs"].(map[string]interface{})
+
+	defaultConf := configs["default"].(map[string]interface{})
+
+	ZabovUpDNS = defaultConf["upstream"].(string)
+	ZabovSingleBL = defaultConf["singlefilters"].(string)
+	ZabovDoubleBL = defaultConf["doublefilters"].(string)
+	ZabovAddBL = defaultConf["blackholeip"].(string)
+	ZabovHostsFile = defaultConf["hostsfile"].(string)
 
 	zabovString := ZabovAddr + ":" + ZabovPort
 
@@ -66,4 +62,111 @@ func init() {
 
 	ZabovDNSArray = fileByLines(ZabovUpDNS)
 
+	ZabovConfigs = map[string]ZabovConfig{}
+	ZabovIPGroups = []ZabovIPGroup{}
+	ZabovTimetables = map[string]ZabovTimetable{}
+	ZabovIPAliases = map[string]string{}
+	ZabovDNSArrays = map[string][]string{}
+	IPAliasesRaw := MyConf["ipaliases"].(map[string]interface{})
+
+	for alias, ip := range IPAliasesRaw {
+		fmt.Println("IP Alias:", alias, ip)
+		ZabovIPAliases[alias] = ip.(string)
+	}
+
+	for name, v := range configs {
+		fmt.Println("evaluaing config name:", name)
+		confRaw := v.(map[string]interface{})
+		var conf ZabovConfig
+		conf.ZabovUpDNS = confRaw["upstream"].(string)
+		conf.ZabovSingleBL = confRaw["singlefilters"].(string)
+		conf.ZabovDoubleBL = confRaw["doublefilters"].(string)
+		conf.ZabovAddBL = confRaw["blackholeip"].(string)
+		conf.ZabovHostsFile = confRaw["hostsfile"].(string)
+
+		ZabovDNSArrays[name] = fileByLines(conf.ZabovUpDNS)
+		ZabovConfigs[name] = conf
+		if name == "default" {
+			ZabovConfigDefault = conf
+		}
+		ZabovCreateKDB(name)
+	}
+
+	timetables := MyConf["timetables"].(map[string]interface{})
+
+	for name, v := range timetables {
+		fmt.Println("evaluaing timetable name:", name)
+		timetableRaw := v.(map[string]interface{})
+		var timetable ZabovTimetable
+
+		timetable.cfgin = timetableRaw["cfgin"].(string)
+		timetable.cfgout = timetableRaw["cfgout"].(string)
+
+		if timetable.cfgin == "" {
+			timetable.cfgin = "default"
+		}
+		if timetable.cfgout == "" {
+			timetable.cfgout = "default"
+		}
+
+		_, ok := ZabovConfigs[timetable.cfgin]
+		if !ok {
+			log.Println("inexistent cfgin:", timetable.cfgin)
+			os.Exit(1)
+		}
+
+		_, ok = ZabovConfigs[timetable.cfgout]
+		if !ok {
+			log.Println("inexistent cfgout:", timetable.cfgout)
+			os.Exit(1)
+		}
+
+		tables := timetableRaw["tables"].([]interface{})
+
+		for i := range tables {
+			table := tables[i].(map[string]interface{})
+			var ttEntry ZabovTimetableEntry
+			ttEntry.times = strings.Split(table["times"].(string), ";")
+			ttEntry.days = strings.Split(table["days"].(string), ";")
+			timetable.table = append(timetable.table, ttEntry)
+		}
+		ZabovTimetables[name] = timetable
+	}
+
+	IPGroups := MyConf["ipgroups"].([]interface{})
+
+	fmt.Println("evaluating IP Groups: ", len(IPGroups))
+	for i := range IPGroups {
+		fmt.Println("evaluating IP Group n.", i)
+		var groupStruct ZabovIPGroup
+		groupMap := IPGroups[i].(map[string]interface{})
+		IPsRaw := groupMap["ips"].([]interface{})
+		groupStruct.ips = []string{}
+		for x := range IPsRaw {
+			ip := IPsRaw[x].(string)
+			fmt.Println("adding IP ", ip)
+
+			alias, ok := ZabovIPAliases[ip]
+			if ok {
+				fmt.Println("IP alias: ", ip, alias)
+				ip = alias
+			}
+			groupStruct.ips = append(groupStruct.ips, ip)
+		}
+		groupStruct.cfg = groupMap["cfg"].(string)
+		groupStruct.timetable = groupMap["timetable"].(string)
+		fmt.Println("cfg:", groupStruct.cfg)
+		fmt.Println("timetable:", groupStruct.timetable)
+		_, ok := ZabovTimetables[groupStruct.timetable]
+		if !ok {
+			log.Println("inexistent timetable:", groupStruct.timetable)
+			os.Exit(1)
+		}
+		ZabovIPGroups = append(ZabovIPGroups, groupStruct)
+	}
+	fmt.Println("ZabovConfigs:", ZabovConfigs)
+	fmt.Println("ZabovTimetables:", ZabovTimetables)
+	fmt.Println("ZabovIPAliases:", ZabovIPAliases)
+	fmt.Println("ZabovIPGroups:", ZabovIPGroups)
+
 }

01.killfile.go

@@ -5,11 +5,10 @@ import (
 	"strings"
 )
 
-var zabovKbucket = []byte("killfile")
-
 type killfileItem struct {
-	Kdomain string
+	Kdomain  string
-	Ksource string
+	Ksource  string
+	Kconfigs stringarray
 }
 
 var bChannel chan killfileItem
@@ -27,7 +26,9 @@ func bWriteThread() {
 
 	for item := range bChannel {
 
-		writeInKillfile(item.Kdomain, item.Ksource)
+		for _, config := range item.Kconfigs {
+			writeInKillfile(item.Kdomain, item.Ksource, config)
+		}
 		incrementStats("BL domains from "+item.Ksource, 1)
 		incrementStats("TOTAL", 1)
 
@@ -36,7 +37,7 @@ func bWriteThread() {
 }
 
 //DomainKill stores a domain name inside the killfile
-func DomainKill(s, durl string) {
+func DomainKill(s, durl string, configs stringarray) {
 
 	if len(s) > 2 {
 
@@ -46,6 +47,7 @@ func DomainKill(s, durl string) {
 
 		k.Kdomain = s
 		k.Ksource = durl
+		k.Kconfigs = configs
 
 		bChannel <- k
 
@@ -53,11 +55,12 @@ func DomainKill(s, durl string) {
 
 }
 
-func writeInKillfile(key, value string) {
+func writeInKillfile(key, value string, config string) {
 
 	stK := []byte(key)
 	stV := []byte(value)
 
+	MyZabovKDB := MyZabovKDBs[config]
 	err := MyZabovKDB.Put(stK, stV, nil)
 	if err != nil {
 		fmt.Println("Cannot write to Killfile DB: ", err.Error())
@@ -65,10 +68,11 @@ func writeInKillfile(key, value string) {
 
 }
 
-func domainInKillfile(domain string) bool {
+func domainInKillfile(domain string, config string) bool {
 
 	s := strings.ToLower(domain)
 
+	MyZabovKDB := MyZabovKDBs[config]
 	has, err := MyZabovKDB.Has([]byte(s), nil)
 	if err != nil {
 		fmt.Println("Cannot read from Killfile DB: ", err.Error())

adlist_hosts.go

@@ -16,9 +16,9 @@ func init() {
 }
 
 //DoubleIndexFilter puts the domains inside file
-func DoubleIndexFilter(durl string) error {
+func DoubleIndexFilter(durl string, configs stringarray) error {
 
-	fmt.Println("Retrieving HostFile from: ", durl)
+	fmt.Println("DoubleIndexFilter: Retrieving HostFile from: ", durl)
 
 	var err error
 
@@ -59,7 +59,8 @@ func DoubleIndexFilter(durl string) error {
 		}
 
 		if net.ParseIP(h[0]) != nil {
-			DomainKill(h[1], durl)
+
+			DomainKill(h[1], durl, configs)
 
 			// fmt.Println("MATCH: ", h[1])
 			numLines++
@@ -76,20 +77,38 @@ func DoubleIndexFilter(durl string) error {
 
 }
 
-func getDoubleFilters() {
+func getDoubleFilters(urls urlsMap) {
 
-	s := fileByLines(ZabovDoubleBL)
+	fmt.Println("getDoubleFilters: downloading all urls:", len(urls))
-
+	for url, configs := range urls {
-	for _, a := range s {
+		DoubleIndexFilter(url, configs)
-		DoubleIndexFilter(a)
 	}
 
 }
 
 func downloadDoubleThread() {
 	fmt.Println("Starting updater of DOUBLE lists, each (hours):", ZabovKillTTL)
+
+	_urls := urlsMap{}
+
 	for {
-		getDoubleFilters()
+		fmt.Println("downloadDoubleThread: collecting urls from all configs...")
+		for config := range ZabovConfigs {
+			ZabovDoubleBL := ZabovConfigs[config].ZabovDoubleBL
+
+			s := fileByLines(ZabovDoubleBL)
+			for _, v := range s {
+				configs := _urls[v]
+				if configs == nil {
+					configs = stringarray{}
+					_urls[v] = configs
+				}
+				configs = append(configs, config)
+				_urls[v] = configs
+			}
+		}
+
+		getDoubleFilters(_urls)
 		time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
 	}
 

adlist_single.go

@@ -14,7 +14,7 @@ func init() {
 }
 
 //SingleIndexFilter puts the domains inside file
-func SingleIndexFilter(durl string) error {
+func SingleIndexFilter(durl string, configs stringarray) error {
 
 	fmt.Println("Retrieving DomainFile from: ", durl)
 
@@ -57,7 +57,9 @@ func SingleIndexFilter(durl string) error {
 		}
 
 		if !strings.Contains(h[0], "#") {
-			DomainKill(h[0], durl)
+
+			DomainKill(h[0], durl, configs)
+
 			// fmt.Println("MATCH: ", h[1])
 			numLines++
 		} else {
@@ -73,20 +75,37 @@ func SingleIndexFilter(durl string) error {
 
 }
 
-func getSingleFilters() {
+func getSingleFilters(urls urlsMap) {
 
-	s := fileByLines(ZabovSingleBL)
+	fmt.Println("getSingleFilters: downloading all urls:", len(urls))
-
+	for url, configs := range urls {
-	for _, a := range s {
+		SingleIndexFilter(url, configs)
-		SingleIndexFilter(a)
 	}
 
 }
 
 func downloadThread() {
 	fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL)
+	_urls := urlsMap{}
+
 	for {
-		getSingleFilters()
+		fmt.Println("downloadThread: collecting urls from all configs...")
+		for config := range ZabovConfigs {
+			ZabovSingleBL := ZabovConfigs[config].ZabovSingleBL
+
+			s := fileByLines(ZabovSingleBL)
+			for _, v := range s {
+				configs := _urls[v]
+				if configs == nil {
+					configs = stringarray{}
+					_urls[v] = configs
+				}
+				configs = append(configs, config)
+				_urls[v] = configs
+			}
+		}
+
+		getSingleFilters(_urls)
 		time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
 	}
 

config.json

@@ -1,15 +1,50 @@
 {
-    "zabov": {  
+    "zabov":{
         "port":"53", 
         "proto":"udp", 
         "ipaddr":"0.0.0.0",
-        "upstream":"./dns-upstream.txt"  ,
         "cachettl": 1,
-        "killfilettl": 12,
+        "killfilettl": 12
-        "singlefilters":"./urls-domains.txt" ,
+    },
-        "doublefilters":"./urls-hosts.txt", 
+    "ipaliases":{
-        "blackholeip":"127.0.0.1",
+        "pc8":"192.168.178.29"
-        "hostsfile":"./urls-local.txt"
+    },
+    "ipgroups":[
+        {
+            "ips":["192.168.178.30/32", "192.168.178.31/32", "pc8"],
+            "cfg":"",
+            "timetable":"tt_children"
+        }
+    ],
+    "timetables":{
+        "tt_children":{
+            "tables":[{"times":"8:30-12:30;18:30-22:30", "days":"Mo;Tu;We;Th;Fr;Sa;Su"}],
+            "cfgin":"children_restricted",
+            "cfgout":"children"
+        }
+    },
+    "configs":{
+        "default":{
+            "upstream":"./dns-upstream.txt",
+            "singlefilters":"./urls-domains.txt",
+            "doublefilters":"./urls-hosts.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local.txt"
+        },
+        "children":{
+            "upstream":"./dns-upstream.txt",
+            "singlefilters":"./urls-domains.txt",
+            "doublefilters":"./urls-hosts.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local.txt"
+        },
+        "children_restricted":{
+            "upstream":"./dns-upstream.txt",
+            "singlefilters":"./urls-domains.txt",
+            "doublefilters":"./urls-hosts.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local.txt"
+        }
     }
 
 }

dns_client.go

@@ -12,7 +12,8 @@ import (
 
 //ForwardQuery forwards the query to the upstream server
 //first server to answer wins
-func ForwardQuery(query *dns.Msg) *dns.Msg {
+//accepts config name to select the UP DNS source list
+func ForwardQuery(query *dns.Msg, config string) *dns.Msg {
 
 	go incrementStats("ForwardQueries", 1)
 
@@ -45,7 +46,7 @@ func ForwardQuery(query *dns.Msg) *dns.Msg {
 			continue
 		}
 
-		d := oneTimeDNS()
+		d := oneTimeDNS(config)
 
 		in, _, err := c.Exchange(query, d)
 		if err != nil {
@@ -78,10 +79,11 @@ func init() {
 
 }
 
-func oneTimeDNS() (dns string) {
+func oneTimeDNS(config string) (dns string) {
 
 	rand.Seed(time.Now().Unix())
 
+	ZabovDNSArray := ZabovDNSArrays[config]
 	upl := ZabovDNSArray
 
 	if len(upl) < 1 {

dns_handler.go

@@ -12,19 +12,23 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 
 	remIP, _, e := net.SplitHostPort(w.RemoteAddr().String())
 	if e != nil {
+		go incrementStats("CLIENT ERROR: "+remIP, 1)
+	} else {
 		go incrementStats("CLIENT: "+remIP, 1)
 	}
 
 	msg := dns.Msg{}
 	msg.SetReply(r)
 
+	config := "default" // TODO: get config from client IP & timetable
+
 	switch r.Question[0].Qtype {
 	case dns.TypeA:
 		msg.Authoritative = true
 		domain := msg.Question[0].Name
 		fqdn := strings.TrimRight(domain, ".")
 
-		if domainInKillfile(fqdn) {
+		if domainInKillfile(fqdn, config) {
 			go incrementStats("Killed", 1)
 
 			msg.Answer = append(msg.Answer, &dns.A{
@@ -32,11 +36,11 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 				A:   net.ParseIP(ZabovAddBL),
 			})
 		} else {
-			ret := ForwardQuery(r)
+			ret := ForwardQuery(r, config)
 			w.WriteMsg(ret)
 		}
 	default:
-		ret := ForwardQuery(r)
+		ret := ForwardQuery(r, config)
 		w.WriteMsg(ret)
 	}
 	w.WriteMsg(&msg)

hostfile.go

@@ -9,28 +9,43 @@ import (
 func init() {
 
 	fmt.Println("Ingesting local hosts file")
-	ingestLocalBlacklist()
+	ingestLocalBlacklists()
 
 }
 
-func ingestLocalBlacklist() {
+func ingestLocalBlacklists() {
-
-	file, err := os.Open(ZabovHostsFile)
-	if err != nil {
-		fmt.Println(err.Error())
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		d := scanner.Text()
-		DomainKill(d, ZabovHostsFile)
-		incrementStats("Blacklist", 1)
 
+	fmt.Println("ingestLocalBlacklist: collecting urls from all configs...")
+	_files := urlsMap{}
+	for config := range ZabovConfigs {
+		ZabovHostsFile := ZabovConfigs[config].ZabovHostsFile
+		configs := _files[ZabovHostsFile]
+		if configs == nil {
+			configs = stringarray{}
+			_files[ZabovHostsFile] = configs
+		}
+		configs = append(configs, config)
+		_files[ZabovHostsFile] = configs
 	}
 
-	if err := scanner.Err(); err != nil {
+	for ZabovHostsFile, configs := range _files {
-		fmt.Println(err.Error())
+		file, err := os.Open(ZabovHostsFile)
+		if err != nil {
+			fmt.Println(err.Error())
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			d := scanner.Text()
+			DomainKill(d, ZabovHostsFile, configs)
+			incrementStats("Blacklist", 1)
+
+		}
+
+		if err := scanner.Err(); err != nil {
+			fmt.Println(err.Error())
+		}
 	}
 
 }

main.go

@@ -35,6 +35,53 @@ var ZabovDNSArray []string
 
 type handler struct{}
 
+//ZabovDNSArrays contains the arrays containing all the DNS we mention
+var ZabovDNSArrays map[string][]string
+
+// ZabovConfig contains all Zabov configs
+type ZabovConfig struct {
+	ZabovUpDNS     string // json:upstream -> ZabovDNSArray
+	ZabovSingleBL  string // json:singlefilters
+	ZabovDoubleBL  string // json:doublefilters
+	ZabovAddBL     string // json:blackholeip
+	ZabovHostsFile string // json:hostsfile
+}
+
+// ZabovConfigs contains all Zabov configs
+var ZabovConfigs map[string]ZabovConfig
+
+// ZabovConfigDefault contains only "default" config
+var ZabovConfigDefault ZabovConfig
+
+// ZabovIPGroup contains Zabov groups of IPs
+type ZabovIPGroup struct {
+	ips       []string // IPs in this group
+	cfg       string   // config name to be used if there is no timetable
+	timetable string   // timetable name to be used for this group; timetable SHALL reference to config name to use
+}
+
+// ZabovIPGroups contains an array of all Zabov groups of IP rules
+var ZabovIPGroups []ZabovIPGroup
+
+// ZabovTimetableEntry contains Zabov single time table entry
+type ZabovTimetableEntry struct {
+	times []string
+	days  []string
+}
+
+// ZabovTimetable contains a Zabov time table
+type ZabovTimetable struct {
+	table  []ZabovTimetableEntry
+	cfgin  string // configuration name to be used if "inside" timetable
+	cfgout string // configuration name to be used if "outiside" timetable
+}
+
+// ZabovTimetables contains all Zabov time tables, by name
+var ZabovTimetables map[string]ZabovTimetable
+
+// ZabovIPAliases contains an array of all Zabov IP aliases
+var ZabovIPAliases map[string]string
+
 func main() {
 
 	MyDNS.Handler = &handler{}