mirror of https://git.keinpfusch.net/loweel/zabov
- added simple stats python script
- added WhiteList database - updated license to EUPL
This commit is contained in:
bloved
parent
f018377b44
commit
ef9f912cf7
|
|
@ -13,6 +13,9 @@ var MyZabovCDB *leveldb.DB
|
|||
//MyZabovKDBs is the storage where we'll put domains to block (one for each config)
|
||||
var MyZabovKDBs map[string]*leveldb.DB
|
||||
|
||||
//MyZabovWLDBs is the storage where we'll put domains to whitelist (one for each config)
|
||||
var MyZabovWLDBs map[string]*leveldb.DB
|
||||
|
||||
func init() {
|
||||
|
||||
var err error
|
||||
|
|
@ -29,6 +32,7 @@ func init() {
|
|||
}
|
||||
|
||||
MyZabovKDBs = map[string]*leveldb.DB{}
|
||||
MyZabovWLDBs = map[string]*leveldb.DB{}
|
||||
}
|
||||
|
||||
// ZabovCreateKDB creates Kill DBs
|
||||
|
|
@ -46,3 +50,19 @@ func ZabovCreateKDB(conf string) {
|
|||
MyZabovKDBs[conf] = KDB
|
||||
|
||||
}
|
||||
|
||||
// ZabovCreateWLDB creates Whitelist DBs
|
||||
func ZabovCreateWLDB(conf string) {
|
||||
var err error
|
||||
|
||||
dbname := "./db/whitelist_" + conf
|
||||
WLDB, err := leveldb.OpenFile(dbname, nil)
|
||||
if err != nil {
|
||||
fmt.Println("Cannot create whitelist db: ", err.Error())
|
||||
} else {
|
||||
fmt.Println("whitelist DB created:", dbname)
|
||||
}
|
||||
|
||||
MyZabovWLDBs[conf] = WLDB
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -111,7 +111,13 @@ func init() {
|
|||
conf.ZabovSingleBL = confRaw["singlefilters"].(string)
|
||||
conf.ZabovDoubleBL = confRaw["doublefilters"].(string)
|
||||
conf.ZabovAddBL = net.ParseIP(confRaw["blackholeip"].(string))
|
||||
conf.ZabovHostsFile = confRaw["hostsfile"].(string)
|
||||
if confRaw["hostsfile"] != nil {
|
||||
conf.ZabovHostsFile = confRaw["hostsfile"].(string)
|
||||
}
|
||||
|
||||
if confRaw["whitelist"] != nil {
|
||||
conf.ZabovWhiteList = confRaw["whitelist"].(string)
|
||||
}
|
||||
|
||||
if confRaw["cache"] != nil {
|
||||
conf.ZabovCache = confRaw["cache"].(bool)
|
||||
|
|
@ -295,6 +301,7 @@ func init() {
|
|||
delete(ZabovConfigs, name)
|
||||
} else {
|
||||
ZabovCreateKDB(name)
|
||||
ZabovCreateWLDB(name)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,20 +11,23 @@ type killfileItem struct {
|
|||
Kconfigs stringarray
|
||||
}
|
||||
|
||||
var bChannel chan killfileItem
|
||||
var bKillChannel chan killfileItem
|
||||
var bWhiteListChannel chan killfileItem
|
||||
|
||||
func init() {
|
||||
|
||||
bChannel = make(chan killfileItem, 1024)
|
||||
bKillChannel = make(chan killfileItem, 1024)
|
||||
bWhiteListChannel = make(chan killfileItem, 1024)
|
||||
fmt.Println("Initializing kill channel engine.")
|
||||
|
||||
go bWriteThread()
|
||||
go bWriteKillThread()
|
||||
go bWriteWhiteListThread()
|
||||
|
||||
}
|
||||
|
||||
func bWriteThread() {
|
||||
func bWriteKillThread() {
|
||||
|
||||
for item := range bChannel {
|
||||
for item := range bKillChannel {
|
||||
|
||||
alreadyInSomeDB := false
|
||||
|
||||
|
|
@ -56,7 +59,7 @@ func DomainKill(s, durl string, configs stringarray) {
|
|||
k.Ksource = durl
|
||||
k.Kconfigs = configs
|
||||
|
||||
bChannel <- k
|
||||
bKillChannel <- k
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -88,3 +91,69 @@ func domainInKillfile(domain string, config string) bool {
|
|||
return has
|
||||
|
||||
}
|
||||
|
||||
func bWriteWhiteListThread() {
|
||||
|
||||
for item := range bWhiteListChannel {
|
||||
|
||||
alreadyInSomeDB := false
|
||||
|
||||
for _, config := range item.Kconfigs {
|
||||
if !alreadyInSomeDB {
|
||||
alreadyInSomeDB = domainInWhiteListfile(item.Kdomain, config)
|
||||
}
|
||||
writeInWhiteListfile(item.Kdomain, item.Ksource, config)
|
||||
}
|
||||
if !alreadyInSomeDB {
|
||||
incrementStats("WL domains from "+item.Ksource, 1)
|
||||
incrementStats("WL TOTAL", 1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//DomainWhiteList stores a domain name inside the killfile
|
||||
func DomainWhiteList(s, durl string, configs stringarray) {
|
||||
|
||||
if len(s) > 2 {
|
||||
|
||||
s = strings.ToLower(s)
|
||||
|
||||
var k killfileItem
|
||||
|
||||
k.Kdomain = s
|
||||
k.Ksource = durl
|
||||
k.Kconfigs = configs
|
||||
|
||||
bWhiteListChannel <- k
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func writeInWhiteListfile(key, value string, config string) {
|
||||
|
||||
stK := []byte(key)
|
||||
stV := []byte(value)
|
||||
|
||||
MyZabovWLDB := MyZabovWLDBs[config]
|
||||
err := MyZabovWLDB.Put(stK, stV, nil)
|
||||
if err != nil {
|
||||
fmt.Println("Cannot write to Whitelist DB: ", err.Error())
|
||||
}
|
||||
|
||||
}
|
||||
func domainInWhiteListfile(domain string, config string) bool {
|
||||
|
||||
s := strings.ToLower(domain)
|
||||
|
||||
MyZabovWLDB := MyZabovWLDBs[config]
|
||||
has, err := MyZabovWLDB.Has([]byte(s), nil)
|
||||
if err != nil {
|
||||
fmt.Println("Cannot read from Whitelist DB: ", err.Error())
|
||||
}
|
||||
|
||||
return has
|
||||
|
||||
}
|
||||
|
|
|
|||
297
LICENSE
297
LICENSE
|
|
@ -1,14 +1,289 @@
|
|||
Copyright (C) 2020 loweel@keinpfusch.net
|
||||
Copyright (C) 2020 loweel@keinpfusch.net
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
EUROPEAN UNION PUBLIC LICENCE v. 1.2
|
||||
EUPL © the European Union 2007, 2016
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined
|
||||
below) which is provided under the terms of this Licence. Any use of the Work,
|
||||
other than as authorised under this Licence is prohibited (to the extent such
|
||||
use is covered by a right of the copyright holder of the Work).
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
The Work is provided under the terms of this Licence when the Licensor (as
|
||||
defined below) has placed the following notice immediately following the
|
||||
copyright notice for the Work:
|
||||
|
||||
Licensed under the EUPL
|
||||
|
||||
or has expressed by any other means his willingness to license under the EUPL.
|
||||
|
||||
1. Definitions
|
||||
|
||||
In this Licence, the following terms have the following meaning:
|
||||
|
||||
- ‘The Licence’: this Licence.
|
||||
|
||||
- ‘The Original Work’: the work or software distributed or communicated by the
|
||||
Licensor under this Licence, available as Source Code and also as Executable
|
||||
Code as the case may be.
|
||||
|
||||
- ‘Derivative Works’: the works or software that could be created by the
|
||||
Licensee, based upon the Original Work or modifications thereof. This Licence
|
||||
does not define the extent of modification or dependence on the Original Work
|
||||
required in order to classify a work as a Derivative Work; this extent is
|
||||
determined by copyright law applicable in the country mentioned in Article 15.
|
||||
|
||||
- ‘The Work’: the Original Work or its Derivative Works.
|
||||
|
||||
- ‘The Source Code’: the human-readable form of the Work which is the most
|
||||
convenient for people to study and modify.
|
||||
|
||||
- ‘The Executable Code’: any code which has generally been compiled and which is
|
||||
meant to be interpreted by a computer as a program.
|
||||
|
||||
- ‘The Licensor’: the natural or legal person that distributes or communicates
|
||||
the Work under the Licence.
|
||||
|
||||
- ‘Contributor(s)’: any natural or legal person who modifies the Work under the
|
||||
Licence, or otherwise contributes to the creation of a Derivative Work.
|
||||
|
||||
- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of
|
||||
the Work under the terms of the Licence.
|
||||
|
||||
- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending,
|
||||
renting, distributing, communicating, transmitting, or otherwise making
|
||||
available, online or offline, copies of the Work or providing access to its
|
||||
essential functionalities at the disposal of any other natural or legal
|
||||
person.
|
||||
|
||||
2. Scope of the rights granted by the Licence
|
||||
|
||||
The Licensor hereby grants You a worldwide, royalty-free, non-exclusive,
|
||||
sublicensable licence to do the following, for the duration of copyright vested
|
||||
in the Original Work:
|
||||
|
||||
- use the Work in any circumstance and for all usage,
|
||||
- reproduce the Work,
|
||||
- modify the Work, and make Derivative Works based upon the Work,
|
||||
- communicate to the public, including the right to make available or display
|
||||
the Work or copies thereof to the public and perform publicly, as the case may
|
||||
be, the Work,
|
||||
- distribute the Work or copies thereof,
|
||||
- lend and rent the Work or copies thereof,
|
||||
- sublicense rights in the Work or copies thereof.
|
||||
|
||||
Those rights can be exercised on any media, supports and formats, whether now
|
||||
known or later invented, as far as the applicable law permits so.
|
||||
|
||||
In the countries where moral rights apply, the Licensor waives his right to
|
||||
exercise his moral right to the extent allowed by law in order to make effective
|
||||
the licence of the economic rights here above listed.
|
||||
|
||||
The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to
|
||||
any patents held by the Licensor, to the extent necessary to make use of the
|
||||
rights granted on the Work under this Licence.
|
||||
|
||||
3. Communication of the Source Code
|
||||
|
||||
The Licensor may provide the Work either in its Source Code form, or as
|
||||
Executable Code. If the Work is provided as Executable Code, the Licensor
|
||||
provides in addition a machine-readable copy of the Source Code of the Work
|
||||
along with each copy of the Work that the Licensor distributes or indicates, in
|
||||
a notice following the copyright notice attached to the Work, a repository where
|
||||
the Source Code is easily and freely accessible for as long as the Licensor
|
||||
continues to distribute or communicate the Work.
|
||||
|
||||
4. Limitations on copyright
|
||||
|
||||
Nothing in this Licence is intended to deprive the Licensee of the benefits from
|
||||
any exception or limitation to the exclusive rights of the rights owners in the
|
||||
Work, of the exhaustion of those rights or of other applicable limitations
|
||||
thereto.
|
||||
|
||||
5. Obligations of the Licensee
|
||||
|
||||
The grant of the rights mentioned above is subject to some restrictions and
|
||||
obligations imposed on the Licensee. Those obligations are the following:
|
||||
|
||||
Attribution right: The Licensee shall keep intact all copyright, patent or
|
||||
trademarks notices and all notices that refer to the Licence and to the
|
||||
disclaimer of warranties. The Licensee must include a copy of such notices and a
|
||||
copy of the Licence with every copy of the Work he/she distributes or
|
||||
communicates. The Licensee must cause any Derivative Work to carry prominent
|
||||
notices stating that the Work has been modified and the date of modification.
|
||||
|
||||
Copyleft clause: If the Licensee distributes or communicates copies of the
|
||||
Original Works or Derivative Works, this Distribution or Communication will be
|
||||
done under the terms of this Licence or of a later version of this Licence
|
||||
unless the Original Work is expressly distributed only under this version of the
|
||||
Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee
|
||||
(becoming Licensor) cannot offer or impose any additional terms or conditions on
|
||||
the Work or Derivative Work that alter or restrict the terms of the Licence.
|
||||
|
||||
Compatibility clause: If the Licensee Distributes or Communicates Derivative
|
||||
Works or copies thereof based upon both the Work and another work licensed under
|
||||
a Compatible Licence, this Distribution or Communication can be done under the
|
||||
terms of this Compatible Licence. For the sake of this clause, ‘Compatible
|
||||
Licence’ refers to the licences listed in the appendix attached to this Licence.
|
||||
Should the Licensee's obligations under the Compatible Licence conflict with
|
||||
his/her obligations under this Licence, the obligations of the Compatible
|
||||
Licence shall prevail.
|
||||
|
||||
Provision of Source Code: When distributing or communicating copies of the Work,
|
||||
the Licensee will provide a machine-readable copy of the Source Code or indicate
|
||||
a repository where this Source will be easily and freely available for as long
|
||||
as the Licensee continues to distribute or communicate the Work.
|
||||
|
||||
Legal Protection: This Licence does not grant permission to use the trade names,
|
||||
trademarks, service marks, or names of the Licensor, except as required for
|
||||
reasonable and customary use in describing the origin of the Work and
|
||||
reproducing the content of the copyright notice.
|
||||
|
||||
6. Chain of Authorship
|
||||
|
||||
The original Licensor warrants that the copyright in the Original Work granted
|
||||
hereunder is owned by him/her or licensed to him/her and that he/she has the
|
||||
power and authority to grant the Licence.
|
||||
|
||||
Each Contributor warrants that the copyright in the modifications he/she brings
|
||||
to the Work are owned by him/her or licensed to him/her and that he/she has the
|
||||
power and authority to grant the Licence.
|
||||
|
||||
Each time You accept the Licence, the original Licensor and subsequent
|
||||
Contributors grant You a licence to their contributions to the Work, under the
|
||||
terms of this Licence.
|
||||
|
||||
7. Disclaimer of Warranty
|
||||
|
||||
The Work is a work in progress, which is continuously improved by numerous
|
||||
Contributors. It is not a finished work and may therefore contain defects or
|
||||
‘bugs’ inherent to this type of development.
|
||||
|
||||
For the above reason, the Work is provided under the Licence on an ‘as is’ basis
|
||||
and without warranties of any kind concerning the Work, including without
|
||||
limitation merchantability, fitness for a particular purpose, absence of defects
|
||||
or errors, accuracy, non-infringement of intellectual property rights other than
|
||||
copyright as stated in Article 6 of this Licence.
|
||||
|
||||
This disclaimer of warranty is an essential part of the Licence and a condition
|
||||
for the grant of any rights to the Work.
|
||||
|
||||
8. Disclaimer of Liability
|
||||
|
||||
Except in the cases of wilful misconduct or damages directly caused to natural
|
||||
persons, the Licensor will in no event be liable for any direct or indirect,
|
||||
material or moral, damages of any kind, arising out of the Licence or of the use
|
||||
of the Work, including without limitation, damages for loss of goodwill, work
|
||||
stoppage, computer failure or malfunction, loss of data or any commercial
|
||||
damage, even if the Licensor has been advised of the possibility of such damage.
|
||||
However, the Licensor will be liable under statutory product liability laws as
|
||||
far such laws apply to the Work.
|
||||
|
||||
9. Additional agreements
|
||||
|
||||
While distributing the Work, You may choose to conclude an additional agreement,
|
||||
defining obligations or services consistent with this Licence. However, if
|
||||
accepting obligations, You may act only on your own behalf and on your sole
|
||||
responsibility, not on behalf of the original Licensor or any other Contributor,
|
||||
and only if You agree to indemnify, defend, and hold each Contributor harmless
|
||||
for any liability incurred by, or claims asserted against such Contributor by
|
||||
the fact You have accepted any warranty or additional liability.
|
||||
|
||||
10. Acceptance of the Licence
|
||||
|
||||
The provisions of this Licence can be accepted by clicking on an icon ‘I agree’
|
||||
placed under the bottom of a window displaying the text of this Licence or by
|
||||
affirming consent in any other similar way, in accordance with the rules of
|
||||
applicable law. Clicking on that icon indicates your clear and irrevocable
|
||||
acceptance of this Licence and all of its terms and conditions.
|
||||
|
||||
Similarly, you irrevocably accept this Licence and all of its terms and
|
||||
conditions by exercising any rights granted to You by Article 2 of this Licence,
|
||||
such as the use of the Work, the creation by You of a Derivative Work or the
|
||||
Distribution or Communication by You of the Work or copies thereof.
|
||||
|
||||
11. Information to the public
|
||||
|
||||
In case of any Distribution or Communication of the Work by means of electronic
|
||||
communication by You (for example, by offering to download the Work from a
|
||||
remote location) the distribution channel or media (for example, a website) must
|
||||
at least provide to the public the information requested by the applicable law
|
||||
regarding the Licensor, the Licence and the way it may be accessible, concluded,
|
||||
stored and reproduced by the Licensee.
|
||||
|
||||
12. Termination of the Licence
|
||||
|
||||
The Licence and the rights granted hereunder will terminate automatically upon
|
||||
any breach by the Licensee of the terms of the Licence.
|
||||
|
||||
Such a termination will not terminate the licences of any person who has
|
||||
received the Work from the Licensee under the Licence, provided such persons
|
||||
remain in full compliance with the Licence.
|
||||
|
||||
13. Miscellaneous
|
||||
|
||||
Without prejudice of Article 9 above, the Licence represents the complete
|
||||
agreement between the Parties as to the Work.
|
||||
|
||||
If any provision of the Licence is invalid or unenforceable under applicable
|
||||
law, this will not affect the validity or enforceability of the Licence as a
|
||||
whole. Such provision will be construed or reformed so as necessary to make it
|
||||
valid and enforceable.
|
||||
|
||||
The European Commission may publish other linguistic versions or new versions of
|
||||
this Licence or updated versions of the Appendix, so far this is required and
|
||||
reasonable, without reducing the scope of the rights granted by the Licence. New
|
||||
versions of the Licence will be published with a unique version number.
|
||||
|
||||
All linguistic versions of this Licence, approved by the European Commission,
|
||||
have identical value. Parties can take advantage of the linguistic version of
|
||||
their choice.
|
||||
|
||||
14. Jurisdiction
|
||||
|
||||
Without prejudice to specific agreement between parties,
|
||||
|
||||
- any litigation resulting from the interpretation of this License, arising
|
||||
between the European Union institutions, bodies, offices or agencies, as a
|
||||
Licensor, and any Licensee, will be subject to the jurisdiction of the Court
|
||||
of Justice of the European Union, as laid down in article 272 of the Treaty on
|
||||
the Functioning of the European Union,
|
||||
|
||||
- any litigation arising between other parties and resulting from the
|
||||
interpretation of this License, will be subject to the exclusive jurisdiction
|
||||
of the competent court where the Licensor resides or conducts its primary
|
||||
business.
|
||||
|
||||
15. Applicable Law
|
||||
|
||||
Without prejudice to specific agreement between parties,
|
||||
|
||||
- this Licence shall be governed by the law of the European Union Member State
|
||||
where the Licensor has his seat, resides or has his registered office,
|
||||
|
||||
- this licence shall be governed by Belgian law if the Licensor has no seat,
|
||||
residence or registered office inside a European Union Member State.
|
||||
|
||||
Appendix
|
||||
|
||||
‘Compatible Licences’ according to Article 5 EUPL are:
|
||||
|
||||
- GNU General Public License (GPL) v. 2, v. 3
|
||||
- GNU Affero General Public License (AGPL) v. 3
|
||||
- Open Software License (OSL) v. 2.1, v. 3.0
|
||||
- Eclipse Public License (EPL) v. 1.0
|
||||
- CeCILL v. 2.0, v. 2.1
|
||||
- Mozilla Public Licence (MPL) v. 2
|
||||
- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3
|
||||
- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for
|
||||
works other than software
|
||||
- European Union Public Licence (EUPL) v. 1.1, v. 1.2
|
||||
- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong
|
||||
Reciprocity (LiLiQ-R+).
|
||||
|
||||
The European Commission may update this Appendix to later versions of the above
|
||||
licences without producing a new version of the EUPL, as long as they provide
|
||||
the rights granted in Article 2 of this Licence and protect the covered Source
|
||||
Code from exclusive appropriation.
|
||||
|
||||
All other changes or additions to this Appendix require the production of a new
|
||||
EUPL version.
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
|
|||
}
|
||||
|
||||
}
|
||||
if domainInKillfile(fqdn, config) {
|
||||
if !domainInWhiteListfile(fqdn, config) && domainInKillfile(fqdn, config) {
|
||||
go incrementStats("Killed", 1)
|
||||
|
||||
msg.Answer = append(msg.Answer, &dns.A{
|
||||
|
|
|
|||
54
hostfile.go
54
hostfile.go
|
|
@ -11,28 +11,29 @@ func init() {
|
|||
|
||||
fmt.Println("Ingesting local hosts file")
|
||||
ingestLocalBlacklists()
|
||||
ingestLocalWhiteLists()
|
||||
|
||||
}
|
||||
|
||||
func ingestLocalBlacklists() {
|
||||
|
||||
fmt.Println("ingestLocalBlacklist: collecting urls from all configs...")
|
||||
_files := urlsMap{}
|
||||
_HostsFiles := urlsMap{}
|
||||
for config := range ZabovConfigs {
|
||||
ZabovHostsFile := ZabovConfigs[config].ZabovHostsFile
|
||||
if len(ZabovHostsFile) == 0 {
|
||||
continue
|
||||
}
|
||||
configs := _files[ZabovHostsFile]
|
||||
configs := _HostsFiles[ZabovHostsFile]
|
||||
if configs == nil {
|
||||
configs = stringarray{}
|
||||
_files[ZabovHostsFile] = configs
|
||||
_HostsFiles[ZabovHostsFile] = configs
|
||||
}
|
||||
configs = append(configs, config)
|
||||
_files[ZabovHostsFile] = configs
|
||||
_HostsFiles[ZabovHostsFile] = configs
|
||||
}
|
||||
|
||||
for ZabovHostsFile, configs := range _files {
|
||||
for ZabovHostsFile, configs := range _HostsFiles {
|
||||
file, err := os.Open(ZabovHostsFile)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
|
|
@ -57,6 +58,49 @@ func ingestLocalBlacklists() {
|
|||
|
||||
}
|
||||
|
||||
func ingestLocalWhiteLists() {
|
||||
|
||||
fmt.Println("ingestLocalWhiteLists: collecting urls from all configs...")
|
||||
_WhiteListFiles := urlsMap{}
|
||||
for config := range ZabovConfigs {
|
||||
ZabovWhiteList := ZabovConfigs[config].ZabovWhiteList
|
||||
if len(ZabovWhiteList) == 0 {
|
||||
continue
|
||||
}
|
||||
configs := _WhiteListFiles[ZabovWhiteList]
|
||||
if configs == nil {
|
||||
configs = stringarray{}
|
||||
_WhiteListFiles[ZabovWhiteList] = configs
|
||||
}
|
||||
configs = append(configs, config)
|
||||
_WhiteListFiles[ZabovWhiteList] = configs
|
||||
}
|
||||
|
||||
for ZabovWhiteList, configs := range _WhiteListFiles {
|
||||
file, err := os.Open(ZabovWhiteList)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
d := scanner.Text()
|
||||
if len(d) == 0 || strings.TrimSpace(d)[0] == '#' {
|
||||
continue
|
||||
}
|
||||
DomainWhiteList(d, ZabovWhiteList, configs)
|
||||
incrementStats("WhiteList", 1)
|
||||
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
fmt.Println(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func fileByLines(filename string) (blurls []string) {
|
||||
|
||||
file, err := os.Open(filename)
|
||||
|
|
|
|||
1
main.go
1
main.go
|
|
@ -39,6 +39,7 @@ type ZabovConfig struct {
|
|||
ZabovDoubleBL string // json:doublefilters -> ZabovDoubleBL list of urls returning a file with IP<space>domain
|
||||
ZabovAddBL net.IP // json:blackholeip -> ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1
|
||||
ZabovHostsFile string // json:hostsfile -> ZabovHostsFile is the file we use to keep our hosts
|
||||
ZabovWhiteList string // json:hostsfile -> ZabovWhiteList is the file we use to keep white listed hosts
|
||||
ZabovUpDNS string // json:upstream -> ZabovUpDNS keeps the name of upstream DNSs
|
||||
ZabovDNSArray []string // contains all the DNS we mention, parsed from ZabovUpDNS file
|
||||
ZabovCache bool // allows to disable cache
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
#!/bin/env python3
|
||||
|
||||
import os
|
||||
import glob
|
||||
import argparse
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description='Zabov logs analyzer')
|
||||
|
||||
parser.add_argument('--ip', dest="ip", metavar='IP', type=str,
|
||||
help='filter by source IP (substring to match). Default: any')
|
||||
|
||||
parser.add_argument('--action', dest="action", metavar='action', type=str, default="killed",
|
||||
help='filter action (substring to match): killed|forwarded|any. Default: killed')
|
||||
|
||||
parser.add_argument('--config', dest="config", metavar='name', type=str,
|
||||
help='filter by config name (substring to match). Default: any')
|
||||
|
||||
parser.add_argument('--timetable', dest="timetable", metavar='name', type=str,
|
||||
help='filter by timetable name (substring to match). Default: any')
|
||||
|
||||
parser.add_argument('--reqtype', dest="reqtype", metavar='TypeA', type=str, default="TypeA",
|
||||
help='filter by reqtype name (substring to match): TypeA|TypeAAAA|TypeMX|...')
|
||||
|
||||
parser.add_argument('--domain', dest="domain", metavar='name', type=str,
|
||||
help='filter by domain name (substring to match). Default: all')
|
||||
|
||||
parser.add_argument('--min-entries', dest="minentries", metavar='100', type=int, default=0,
|
||||
help='filter output by minimum number of entries. Default: any')
|
||||
|
||||
parser.add_argument('--logs-path', dest="logs", metavar='path', type=str, default="./config/logs",
|
||||
help='Zabov logs path')
|
||||
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
timetables = {}
|
||||
configs = {}
|
||||
|
||||
killed = {}
|
||||
for x in glob.glob(os.path.join(args.logs, "*.log")):
|
||||
#print (x)
|
||||
f = open(x, "r")
|
||||
f.readline()
|
||||
if args.reqtype:
|
||||
args.reqtype = args.reqtype.lower()
|
||||
if args.domain:
|
||||
args.domain = args.domain.lower()
|
||||
|
||||
for line in f.readlines():
|
||||
linel = line.strip().lower()
|
||||
fields = linel.split("\t")
|
||||
|
||||
timetables[fields[5]] = timetables.get(fields[5], 0) +1
|
||||
configs[fields[4]] = configs.get(fields[4], 0) +1
|
||||
|
||||
ok = all((not args.action or fields[6].find(args.action)>=0 or fields[6] == "any", \
|
||||
not args.timetable or fields[5].find(args.timetable)>=0 or fields[5] == "any", \
|
||||
not args.config or fields[4].find(args.config)>=0 or fields[4] == "any", \
|
||||
not args.ip or fields[1].find(args.ip)>=0 or fields[1] == "any",\
|
||||
not args.domain or fields[2].find(args.domain)>=0 or fields[2] == "any", \
|
||||
not args.reqtype or fields[3].find(args.reqtype)>=0 ))
|
||||
if ok:
|
||||
killed[fields[2]] = killed.get(fields[2], 0) +1
|
||||
|
||||
killed_sorted = {key: value for key, value in sorted(killed.items(), key=lambda item: item[1], reverse=True)}
|
||||
|
||||
total_queries_filtered = 0
|
||||
total_domain_filtered = 0
|
||||
total_queries = 0
|
||||
for k in killed_sorted.keys():
|
||||
if args.minentries == 0 or killed[k] >= args.minentries:
|
||||
print (k, killed[k])
|
||||
total_queries_filtered += killed[k]
|
||||
total_domain_filtered+=1
|
||||
total_queries += killed[k]
|
||||
|
||||
print("")
|
||||
print("TOTAL domains (filtered):", total_domain_filtered )
|
||||
print("TOTAL queries (filtred):", total_queries_filtered )
|
||||
print("TOTAL domains:", len(killed_sorted.keys()) )
|
||||
print("TOTAL queries:", total_queries )
|
||||
|
||||
|
||||
timetables = {key: value for key, value in sorted(timetables.items(), key=lambda item: item[0], reverse=False)}
|
||||
configs = {key: value for key, value in sorted(configs.items(), key=lambda item: item[0], reverse=False)}
|
||||
|
||||
print("all available timetables:")
|
||||
for k in timetables.keys():
|
||||
print(" '%s': %d items" % (k, timetables[k], ))
|
||||
print("all available configs:")
|
||||
for k in configs.keys():
|
||||
print(" '%s': %d items" % (k, configs[k], ))
|
||||
Loading…
Reference in New Issue