mirror of
https://github.com/agresdominik/f2b_parser.git
synced 2026-04-21 18:05:47 +00:00
code
This commit is contained in:
@@ -0,0 +1,6 @@
|
|||||||
|
charts/
|
||||||
|
test/
|
||||||
|
bin/
|
||||||
|
|
||||||
|
*.json
|
||||||
|
*.log
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
module logs
|
||||||
|
|
||||||
|
go 1.25.3
|
||||||
|
|
||||||
|
require (
|
||||||
|
codeberg.org/go-fonts/liberation v0.5.0 // indirect
|
||||||
|
codeberg.org/go-latex/latex v0.1.0 // indirect
|
||||||
|
codeberg.org/go-pdf/fpdf v0.10.0 // indirect
|
||||||
|
git.sr.ht/~sbinet/gg v0.6.0 // indirect
|
||||||
|
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect
|
||||||
|
github.com/campoy/embedmd v1.0.0 // indirect
|
||||||
|
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
|
golang.org/x/image v0.25.0 // indirect
|
||||||
|
golang.org/x/text v0.23.0 // indirect
|
||||||
|
gonum.org/v1/plot v0.16.0 // indirect
|
||||||
|
rsc.io/pdf v0.1.1 // indirect
|
||||||
|
)
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
codeberg.org/go-fonts/liberation v0.5.0 h1:SsKoMO1v1OZmzkG2DY+7ZkCL9U+rrWI09niOLfQ5Bo0=
|
||||||
|
codeberg.org/go-fonts/liberation v0.5.0/go.mod h1:zS/2e1354/mJ4pGzIIaEtm/59VFCFnYC7YV6YdGl5GU=
|
||||||
|
codeberg.org/go-latex/latex v0.1.0 h1:hoGO86rIbWVyjtlDLzCqZPjNykpWQ9YuTZqAzPcfL3c=
|
||||||
|
codeberg.org/go-latex/latex v0.1.0/go.mod h1:LA0q/AyWIYrqVd+A9Upkgsb+IqPcmSTKc9Dny04MHMw=
|
||||||
|
codeberg.org/go-pdf/fpdf v0.10.0 h1:u+w669foDDx5Ds43mpiiayp40Ov6sZalgcPMDBcZRd4=
|
||||||
|
codeberg.org/go-pdf/fpdf v0.10.0/go.mod h1:Y0DGRAdZ0OmnZPvjbMp/1bYxmIPxm0ws4tfoPOc4LjU=
|
||||||
|
git.sr.ht/~sbinet/gg v0.6.0 h1:RIzgkizAk+9r7uPzf/VfbJHBMKUr0F5hRFxTUGMnt38=
|
||||||
|
git.sr.ht/~sbinet/gg v0.6.0/go.mod h1:uucygbfC9wVPQIfrmwM2et0imr8L7KQWywX0xpFMm94=
|
||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
|
||||||
|
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
|
||||||
|
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw=
|
||||||
|
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
|
||||||
|
github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
|
||||||
|
github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
|
||||||
|
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
|
||||||
|
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||||
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
|
golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
|
||||||
|
golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
|
||||||
|
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||||
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
|
||||||
|
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
gonum.org/v1/plot v0.16.0 h1:dK28Qx/Ky4VmPUN/2zeW0ELyM6ucDnBAj5yun7M9n1g=
|
||||||
|
gonum.org/v1/plot v0.16.0/go.mod h1:Xz6U1yDMi6Ni6aaXILqmVIb6Vro8E+K7Q/GeeH+Pn0c=
|
||||||
|
honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
|
||||||
|
rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4=
|
||||||
|
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||||
+124
@@ -0,0 +1,124 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
type StatsByIp struct {
|
||||||
|
IpAddress string `json:"ipAddress"`
|
||||||
|
TotalLogs int `json:"totalLogs"`
|
||||||
|
TotalFound int `json:"totalFound"`
|
||||||
|
TotalBanned int `json:"totalBanned"`
|
||||||
|
TotalUnbanned int `json:"totalUnbanned"`
|
||||||
|
Country string `json:"county"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func analyseLogs() {
|
||||||
|
|
||||||
|
data, err := os.ReadFile(ParsedJson)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error opening file: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
statsByIp := make(map[string]*StatsByIp)
|
||||||
|
var fileData []Logs
|
||||||
|
err = json.Unmarshal(data, &fileData)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error unmarshaling data: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range fileData {
|
||||||
|
|
||||||
|
ip := entry.IpAddress
|
||||||
|
if ip == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
action := entry.Message
|
||||||
|
|
||||||
|
|
||||||
|
if _, exists := statsByIp[ip]; !exists {
|
||||||
|
country, err := getIpAddressCountry(ip)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Failed getting ip-address with error: %v", err)
|
||||||
|
}
|
||||||
|
statsByIp[ip] = &StatsByIp{
|
||||||
|
IpAddress: ip,
|
||||||
|
Country: country,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
statsByIp[ip].TotalLogs += 1
|
||||||
|
|
||||||
|
switch action {
|
||||||
|
case "Found":
|
||||||
|
statsByIp[ip].TotalFound += 1
|
||||||
|
case "Ban", "already banned":
|
||||||
|
statsByIp[ip].TotalBanned += 1
|
||||||
|
case "Unban":
|
||||||
|
statsByIp[ip].TotalUnbanned += 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonData, err := json.MarshalIndent(statsByIp, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Error marshalling the stats file.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = os.WriteFile(StatsByIPFile, jsonData, 0644)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
func analyseExtractedData() {
|
||||||
|
|
||||||
|
totalCounter := make(map[string]float64)
|
||||||
|
totalBans := make(map[string]float64)
|
||||||
|
totalConnections := make(map[string]float64)
|
||||||
|
|
||||||
|
data, err := os.ReadFile(StatsByIPFile)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error opening file: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var statsByIp map[string]StatsByIp
|
||||||
|
err = json.Unmarshal(data, &statsByIp)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error unmarshaling data: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range statsByIp {
|
||||||
|
country := entry.Country
|
||||||
|
|
||||||
|
if _, exists := totalCounter[country]; !exists {
|
||||||
|
totalCounter[country] = 0
|
||||||
|
} else {
|
||||||
|
totalCounter[country] += 1
|
||||||
|
}
|
||||||
|
if entry.TotalBanned > 0 {
|
||||||
|
if _, exists := totalBans[country]; !exists {
|
||||||
|
totalBans[country] = float64(entry.TotalBanned)
|
||||||
|
} else {
|
||||||
|
totalBans[country] += float64(entry.TotalBanned)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if entry.TotalFound > 0 {
|
||||||
|
if _, exists := totalConnections[country]; !exists {
|
||||||
|
totalConnections[country] = float64(entry.TotalFound)
|
||||||
|
} else {
|
||||||
|
totalConnections[country] += float64(entry.TotalFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
barChart("Individual IPs", "Country", totalCounter)
|
||||||
|
barChart("Total Banned Ips", "Country", totalBans)
|
||||||
|
barChart("Total found connections", "Country", totalConnections)
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
type OkResponseJson struct {
|
||||||
|
Ipaddress string `json:"ipaddress"`
|
||||||
|
Continent_code string `json:"continent_code"`
|
||||||
|
Continent_name string `json:"continent_name"`
|
||||||
|
Country_code string `json:"country_code"`
|
||||||
|
Country_name string `json:"country_name"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func getIpAddressCountry(ipAddress string) (country string, err error) {
|
||||||
|
|
||||||
|
url := fmt.Sprintf("https://api.ipaddress.com/iptocountry?format=json&ip=%s", ipAddress)
|
||||||
|
request, err := http.NewRequest("GET", url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("Failed creating request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &http.Client{}
|
||||||
|
response , err := client.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("Failed making request: %v", err)
|
||||||
|
}
|
||||||
|
defer response.Body.Close()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(response.Body)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("Failed reading body: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var responseFormat OkResponseJson
|
||||||
|
err = json.Unmarshal(body, &responseFormat)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("Failed parsing response body with error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return responseFormat.Country_name, nil
|
||||||
|
|
||||||
|
}
|
||||||
+26
@@ -0,0 +1,26 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type function func()
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
|
||||||
|
//timedRun(parseLogsInJson)
|
||||||
|
//timedRun(analyseLogs)
|
||||||
|
//timedRun(analyseExtractedData)
|
||||||
|
timedRun(starter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func timedRun(fn function) {
|
||||||
|
|
||||||
|
now := time.Now().UnixMilli()
|
||||||
|
fn()
|
||||||
|
after := time.Now().UnixMilli()
|
||||||
|
|
||||||
|
runtime := after - now
|
||||||
|
fmt.Printf("\nTotal runtime: %v", runtime)
|
||||||
|
}
|
||||||
+130
@@ -0,0 +1,130 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
type State struct {
|
||||||
|
Offset int64 `json:"offset"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func starter() {
|
||||||
|
|
||||||
|
destinationDirectory := flag.String("destDir", "", "Destination Directory")
|
||||||
|
flag.StringVar(destinationDirectory, "d", "", "Destination Directory (shorthand)")
|
||||||
|
source := flag.String("source", "", "Source Log File")
|
||||||
|
flag.StringVar(source, "s", "", "Source Log File (shorthand)")
|
||||||
|
|
||||||
|
flag.Usage = func() {
|
||||||
|
fmt.Fprintf(os.Stderr, "Usage: %s --destDir <dir> --source <file>\n", os.Args[0])
|
||||||
|
flag.PrintDefaults()
|
||||||
|
}
|
||||||
|
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if *destinationDirectory == "" || *source == "" {
|
||||||
|
flag.Usage()
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
checkParameters(*destinationDirectory, *source)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* TODO: This function does not check read/write permissions yet
|
||||||
|
*/
|
||||||
|
func checkParameters(destinationDirectory string, source string) {
|
||||||
|
|
||||||
|
|
||||||
|
if _, err := os.Stat(source); os.IsNotExist(err) {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: source file does not exist: %s\n", source)
|
||||||
|
os.Exit(1)
|
||||||
|
} else if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: failed reading source file: %s with error: %s\n", source, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(destinationDirectory); os.IsNotExist(err){
|
||||||
|
err = os.MkdirAll(destinationDirectory, 0755)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: failed creating directory file: %s with error: %s\n", source, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
} else if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: failed reading directory file: %s with error: %s\nDid not try to create.", source, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
stateFile, err := initState(destinationDirectory)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: failed initialising state file: %s\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
parseFile(stateFile, source, destinationDirectory)
|
||||||
|
}
|
||||||
|
|
||||||
|
func initState(destinationDirectory string) (stateFilePath string, err error) {
|
||||||
|
|
||||||
|
stateFile := filepath.Join(destinationDirectory, "state.json")
|
||||||
|
|
||||||
|
if _, err := os.Stat(stateFile); err == nil {
|
||||||
|
return stateFile, nil
|
||||||
|
} else if os.IsNotExist(err) {
|
||||||
|
fmt.Println("No state file found, creating...")
|
||||||
|
}
|
||||||
|
|
||||||
|
state := State{
|
||||||
|
Offset: 0,
|
||||||
|
}
|
||||||
|
data, err := json.MarshalIndent(state, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.WriteFile(stateFile, data, 0644)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("State file initialised: %s\n", stateFile)
|
||||||
|
return stateFile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkState(stateFile string) State {
|
||||||
|
|
||||||
|
data, err := os.ReadFile(stateFile)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error opening file: %v", err)
|
||||||
|
return State{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var state State
|
||||||
|
|
||||||
|
err = json.Unmarshal(data, &state)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error reading state file: %s", err)
|
||||||
|
return State{}
|
||||||
|
}
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
|
||||||
|
func updateState(stateFile string, newState State) error {
|
||||||
|
|
||||||
|
data, err := json.MarshalIndent(newState, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.WriteFile(stateFile, data, 0644)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Logs struct {
|
||||||
|
Timestamp string `json:"timestamp"`
|
||||||
|
Handler string `json:"handler"`
|
||||||
|
Level string `json:"level"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
IpAddress string `json:"ipAddress"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var LogFile string = "./data/fail2ban.log"
|
||||||
|
var ParsedJson string = "./data/json_output.json"
|
||||||
|
var StatsByIPFile string = "./data/stats_by_ip.json"
|
||||||
|
|
||||||
|
|
||||||
|
func parseFile(stateFilePath string, logFilePath string, destinationDirectory string) {
|
||||||
|
|
||||||
|
// Init Parsed Log File Name
|
||||||
|
destinationFilePath := filepath.Join(destinationDirectory, "parsed.json")
|
||||||
|
|
||||||
|
// Load metadata
|
||||||
|
offset := checkState(stateFilePath).Offset
|
||||||
|
|
||||||
|
// Check if the log file has rolled over
|
||||||
|
file, err := os.Open(logFilePath)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error opening file: $%v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
stat, _ := file.Stat()
|
||||||
|
if stat.Size() < offset {
|
||||||
|
offset = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read out existing parsed log file
|
||||||
|
var logs []Logs
|
||||||
|
if data, err := os.ReadFile(destinationFilePath); err == nil && len(data) > 0 {
|
||||||
|
_ = json.Unmarshal(data, &logs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Define regex logic
|
||||||
|
const lenTimestamp = 23
|
||||||
|
dateRegex := regexp.MustCompile(`\d{4}-\d{2}-\d{2}`)
|
||||||
|
handlerRegex := regexp.MustCompile(`fail2ban\.\w+`)
|
||||||
|
ipRegex := regexp.MustCompile(`(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`)
|
||||||
|
levelRegex := regexp.MustCompile(`\s*(?:[A-Z]+)\s+`)
|
||||||
|
serviceRegex := regexp.MustCompile(`\s*(?:\[[a-z]+\])\s+`)
|
||||||
|
actionRegex := regexp.MustCompile(`(Found|already banned|Ban|Unban)`)
|
||||||
|
|
||||||
|
logEntry := Logs{}
|
||||||
|
|
||||||
|
|
||||||
|
_, err = file.Seek(offset, io.SeekStart)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Error going to offset: %s\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the file and append to existing log files
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
for scanner.Scan() {
|
||||||
|
|
||||||
|
line := scanner.Text()
|
||||||
|
|
||||||
|
if len(line) < lenTimestamp {
|
||||||
|
continue
|
||||||
|
} else if !dateRegex.MatchString(line[:lenTimestamp]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
timestamp := line[:lenTimestamp]; timestamp = strings.TrimSpace(timestamp)
|
||||||
|
logString := line[lenTimestamp:]
|
||||||
|
|
||||||
|
ipAddress := strings.TrimSpace(ipRegex.FindString(logString))
|
||||||
|
handler := strings.TrimSpace(handlerRegex.FindString(logString))
|
||||||
|
level := strings.TrimSpace(levelRegex.FindString(logString))
|
||||||
|
service := strings.TrimSpace(serviceRegex.FindString(logString))
|
||||||
|
action := strings.TrimSpace(actionRegex.FindString(logString))
|
||||||
|
|
||||||
|
logEntry.IpAddress = ipAddress
|
||||||
|
logEntry.Timestamp = timestamp
|
||||||
|
logEntry.Handler = handler
|
||||||
|
logEntry.Level = level
|
||||||
|
logEntry.Source = service
|
||||||
|
logEntry.Message = action
|
||||||
|
|
||||||
|
logs = append(logs, logEntry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write parsed content and update metadata
|
||||||
|
jsonData, err := json.MarshalIndent(logs, "", " ")
|
||||||
|
_ = os.WriteFile(destinationFilePath, jsonData, 0644)
|
||||||
|
|
||||||
|
newOffset, _ := file.Seek(0, io.SeekCurrent)
|
||||||
|
newState := State{
|
||||||
|
Offset: newOffset,
|
||||||
|
}
|
||||||
|
updateState(stateFilePath, newState)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseLogsInJson() {
|
||||||
|
|
||||||
|
data, err := os.Open(LogFile)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error opening file: $%v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer data.Close()
|
||||||
|
|
||||||
|
logs := []Logs{}
|
||||||
|
|
||||||
|
const lenTimestamp = 23
|
||||||
|
dateRegex, _ := regexp.Compile(`\d{4}-\d{2}-\d{2}`)
|
||||||
|
handlerRegex, _ := regexp.Compile(`fail2ban\.\w+`)
|
||||||
|
ipRegex, _ := regexp.Compile(`(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`)
|
||||||
|
levelRegex, _ := regexp.Compile(`\s*(?:[A-Z]+)\s+`)
|
||||||
|
serviceRegex, _ := regexp.Compile(`\s*(?:\[[a-z]+\])\s+`)
|
||||||
|
actionRegex, _ := regexp.Compile(`(Found|already banned|Ban|Unban)`)
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(data)
|
||||||
|
logEntry := Logs{}
|
||||||
|
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
|
||||||
|
if len(line) < lenTimestamp {
|
||||||
|
continue
|
||||||
|
} else if !dateRegex.MatchString(line[:lenTimestamp]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
timestamp := line[:lenTimestamp]; timestamp = strings.TrimSpace(timestamp)
|
||||||
|
logString := line[lenTimestamp:]
|
||||||
|
|
||||||
|
ipAddress := strings.TrimSpace(ipRegex.FindString(logString))
|
||||||
|
handler := strings.TrimSpace(handlerRegex.FindString(logString))
|
||||||
|
level := strings.TrimSpace(levelRegex.FindString(logString))
|
||||||
|
service := strings.TrimSpace(serviceRegex.FindString(logString))
|
||||||
|
action := strings.TrimSpace(actionRegex.FindString(logString))
|
||||||
|
|
||||||
|
logEntry.IpAddress = ipAddress
|
||||||
|
logEntry.Timestamp = timestamp
|
||||||
|
logEntry.Handler = handler
|
||||||
|
logEntry.Level = level
|
||||||
|
logEntry.Source = service
|
||||||
|
logEntry.Message = action
|
||||||
|
|
||||||
|
logs = append(logs, logEntry)
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonData, err := json.MarshalIndent(logs, "", " ")
|
||||||
|
err = os.WriteFile(ParsedJson, jsonData, 0644)
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"gonum.org/v1/plot"
|
||||||
|
"gonum.org/v1/plot/plotter"
|
||||||
|
"gonum.org/v1/plot/plotutil"
|
||||||
|
"gonum.org/v1/plot/vg"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
func barChart(action string, variable string, counter map[string]float64) {
|
||||||
|
|
||||||
|
labels := []string{}
|
||||||
|
|
||||||
|
plottingCount := plotter.Values{}
|
||||||
|
for key, value := range counter {
|
||||||
|
if value > 100 {
|
||||||
|
labels = append(labels, key)
|
||||||
|
plottingCount = append(plottingCount, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
p := plot.New()
|
||||||
|
|
||||||
|
p.Title.Text = fmt.Sprintf("%v by %v", action, variable)
|
||||||
|
p.Y.Label.Text = fmt.Sprintf("%v", action)
|
||||||
|
|
||||||
|
w := vg.Points(15)
|
||||||
|
|
||||||
|
barsA, err := plotter.NewBarChart(plottingCount, w)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
//barsA.LineStyle.Width = vg.Length(0)
|
||||||
|
barsA.Color = plotutil.Color(0)
|
||||||
|
barsA.Offset = 0
|
||||||
|
|
||||||
|
p.Add(barsA)
|
||||||
|
p.NominalX(labels...)
|
||||||
|
|
||||||
|
fileName := fmt.Sprintf("%v_%v_barchart.png", action, variable)
|
||||||
|
|
||||||
|
if err := p.Save(12*vg.Inch, 6*vg.Inch, fileName); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user