This commit is contained in:
2025-11-03 18:22:24 +01:00
commit 0f5a46b651
11 changed files with 621 additions and 0 deletions
+6
View File
@@ -0,0 +1,6 @@
charts/
test/
bin/
*.json
*.log
View File
+18
View File
@@ -0,0 +1,18 @@
module logs
go 1.25.3
require (
codeberg.org/go-fonts/liberation v0.5.0 // indirect
codeberg.org/go-latex/latex v0.1.0 // indirect
codeberg.org/go-pdf/fpdf v0.10.0 // indirect
git.sr.ht/~sbinet/gg v0.6.0 // indirect
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect
github.com/campoy/embedmd v1.0.0 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/image v0.25.0 // indirect
golang.org/x/text v0.23.0 // indirect
gonum.org/v1/plot v0.16.0 // indirect
rsc.io/pdf v0.1.1 // indirect
)
+51
View File
@@ -0,0 +1,51 @@
codeberg.org/go-fonts/liberation v0.5.0 h1:SsKoMO1v1OZmzkG2DY+7ZkCL9U+rrWI09niOLfQ5Bo0=
codeberg.org/go-fonts/liberation v0.5.0/go.mod h1:zS/2e1354/mJ4pGzIIaEtm/59VFCFnYC7YV6YdGl5GU=
codeberg.org/go-latex/latex v0.1.0 h1:hoGO86rIbWVyjtlDLzCqZPjNykpWQ9YuTZqAzPcfL3c=
codeberg.org/go-latex/latex v0.1.0/go.mod h1:LA0q/AyWIYrqVd+A9Upkgsb+IqPcmSTKc9Dny04MHMw=
codeberg.org/go-pdf/fpdf v0.10.0 h1:u+w669foDDx5Ds43mpiiayp40Ov6sZalgcPMDBcZRd4=
codeberg.org/go-pdf/fpdf v0.10.0/go.mod h1:Y0DGRAdZ0OmnZPvjbMp/1bYxmIPxm0ws4tfoPOc4LjU=
git.sr.ht/~sbinet/gg v0.6.0 h1:RIzgkizAk+9r7uPzf/VfbJHBMKUr0F5hRFxTUGMnt38=
git.sr.ht/~sbinet/gg v0.6.0/go.mod h1:uucygbfC9wVPQIfrmwM2et0imr8L7KQWywX0xpFMm94=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw=
github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/plot v0.16.0 h1:dK28Qx/Ky4VmPUN/2zeW0ELyM6ucDnBAj5yun7M9n1g=
gonum.org/v1/plot v0.16.0/go.mod h1:Xz6U1yDMi6Ni6aaXILqmVIb6Vro8E+K7Q/GeeH+Pn0c=
honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
+5
View File
@@ -0,0 +1,5 @@
run:
go run ./src/.
buid:
go build -o bin/parser ./src/.
+124
View File
@@ -0,0 +1,124 @@
package main
import (
"encoding/json"
"fmt"
"os"
)
type StatsByIp struct {
IpAddress string `json:"ipAddress"`
TotalLogs int `json:"totalLogs"`
TotalFound int `json:"totalFound"`
TotalBanned int `json:"totalBanned"`
TotalUnbanned int `json:"totalUnbanned"`
Country string `json:"county"`
}
func analyseLogs() {
data, err := os.ReadFile(ParsedJson)
if err != nil {
fmt.Printf("Error opening file: %v", err)
return
}
statsByIp := make(map[string]*StatsByIp)
var fileData []Logs
err = json.Unmarshal(data, &fileData)
if err != nil {
fmt.Printf("Error unmarshaling data: %v", err)
return
}
for _, entry := range fileData {
ip := entry.IpAddress
if ip == "" {
continue
}
action := entry.Message
if _, exists := statsByIp[ip]; !exists {
country, err := getIpAddressCountry(ip)
if err != nil {
fmt.Printf("Failed getting ip-address with error: %v", err)
}
statsByIp[ip] = &StatsByIp{
IpAddress: ip,
Country: country,
}
}
statsByIp[ip].TotalLogs += 1
switch action {
case "Found":
statsByIp[ip].TotalFound += 1
case "Ban", "already banned":
statsByIp[ip].TotalBanned += 1
case "Unban":
statsByIp[ip].TotalUnbanned += 1
}
}
jsonData, err := json.MarshalIndent(statsByIp, "", " ")
if err != nil {
fmt.Println("Error marshalling the stats file.")
return
}
err = os.WriteFile(StatsByIPFile, jsonData, 0644)
}
func analyseExtractedData() {
totalCounter := make(map[string]float64)
totalBans := make(map[string]float64)
totalConnections := make(map[string]float64)
data, err := os.ReadFile(StatsByIPFile)
if err != nil {
fmt.Printf("Error opening file: %v", err)
return
}
var statsByIp map[string]StatsByIp
err = json.Unmarshal(data, &statsByIp)
if err != nil {
fmt.Printf("Error unmarshaling data: %v", err)
return
}
for _, entry := range statsByIp {
country := entry.Country
if _, exists := totalCounter[country]; !exists {
totalCounter[country] = 0
} else {
totalCounter[country] += 1
}
if entry.TotalBanned > 0 {
if _, exists := totalBans[country]; !exists {
totalBans[country] = float64(entry.TotalBanned)
} else {
totalBans[country] += float64(entry.TotalBanned)
}
}
if entry.TotalFound > 0 {
if _, exists := totalConnections[country]; !exists {
totalConnections[country] = float64(entry.TotalFound)
} else {
totalConnections[country] += float64(entry.TotalFound)
}
}
}
barChart("Individual IPs", "Country", totalCounter)
barChart("Total Banned Ips", "Country", totalBans)
barChart("Total found connections", "Country", totalConnections)
}
+46
View File
@@ -0,0 +1,46 @@
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
)
type OkResponseJson struct {
Ipaddress string `json:"ipaddress"`
Continent_code string `json:"continent_code"`
Continent_name string `json:"continent_name"`
Country_code string `json:"country_code"`
Country_name string `json:"country_name"`
}
func getIpAddressCountry(ipAddress string) (country string, err error) {
url := fmt.Sprintf("https://api.ipaddress.com/iptocountry?format=json&ip=%s", ipAddress)
request, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", fmt.Errorf("Failed creating request: %v", err)
}
client := &http.Client{}
response , err := client.Do(request)
if err != nil {
return "", fmt.Errorf("Failed making request: %v", err)
}
defer response.Body.Close()
body, err := io.ReadAll(response.Body)
if err != nil {
return "", fmt.Errorf("Failed reading body: %s", err)
}
var responseFormat OkResponseJson
err = json.Unmarshal(body, &responseFormat)
if err != nil {
return "", fmt.Errorf("Failed parsing response body with error: %v", err)
}
return responseFormat.Country_name, nil
}
+26
View File
@@ -0,0 +1,26 @@
package main
import (
"fmt"
"time"
)
type function func()
func main() {
//timedRun(parseLogsInJson)
//timedRun(analyseLogs)
//timedRun(analyseExtractedData)
timedRun(starter)
}
func timedRun(fn function) {
now := time.Now().UnixMilli()
fn()
after := time.Now().UnixMilli()
runtime := after - now
fmt.Printf("\nTotal runtime: %v", runtime)
}
+130
View File
@@ -0,0 +1,130 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"path/filepath"
)
type State struct {
Offset int64 `json:"offset"`
}
func starter() {
destinationDirectory := flag.String("destDir", "", "Destination Directory")
flag.StringVar(destinationDirectory, "d", "", "Destination Directory (shorthand)")
source := flag.String("source", "", "Source Log File")
flag.StringVar(source, "s", "", "Source Log File (shorthand)")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s --destDir <dir> --source <file>\n", os.Args[0])
flag.PrintDefaults()
}
flag.Parse()
if *destinationDirectory == "" || *source == "" {
flag.Usage()
os.Exit(1)
}
checkParameters(*destinationDirectory, *source)
}
/*
* TODO: This function does not check read/write permissions yet
*/
func checkParameters(destinationDirectory string, source string) {
if _, err := os.Stat(source); os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Error: source file does not exist: %s\n", source)
os.Exit(1)
} else if err != nil {
fmt.Fprintf(os.Stderr, "Error: failed reading source file: %s with error: %s\n", source, err)
os.Exit(1)
}
if _, err := os.Stat(destinationDirectory); os.IsNotExist(err){
err = os.MkdirAll(destinationDirectory, 0755)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: failed creating directory file: %s with error: %s\n", source, err)
os.Exit(1)
}
} else if err != nil {
fmt.Fprintf(os.Stderr, "Error: failed reading directory file: %s with error: %s\nDid not try to create.", source, err)
os.Exit(1)
}
stateFile, err := initState(destinationDirectory)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: failed initialising state file: %s\n", err)
os.Exit(1)
}
parseFile(stateFile, source, destinationDirectory)
}
func initState(destinationDirectory string) (stateFilePath string, err error) {
stateFile := filepath.Join(destinationDirectory, "state.json")
if _, err := os.Stat(stateFile); err == nil {
return stateFile, nil
} else if os.IsNotExist(err) {
fmt.Println("No state file found, creating...")
}
state := State{
Offset: 0,
}
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
return "", err
}
err = os.WriteFile(stateFile, data, 0644)
if err != nil {
return "", err
}
fmt.Printf("State file initialised: %s\n", stateFile)
return stateFile, nil
}
func checkState(stateFile string) State {
data, err := os.ReadFile(stateFile)
if err != nil {
fmt.Printf("Error opening file: %v", err)
return State{}
}
var state State
err = json.Unmarshal(data, &state)
if err != nil {
fmt.Fprintf(os.Stderr, "Error reading state file: %s", err)
return State{}
}
return state
}
func updateState(stateFile string, newState State) error {
data, err := json.MarshalIndent(newState, "", " ")
if err != nil {
return err
}
err = os.WriteFile(stateFile, data, 0644)
if err != nil {
return err
}
return nil
}
+167
View File
@@ -0,0 +1,167 @@
package main
import (
"bufio"
"encoding/json"
"fmt"
"io"
"log"
"os"
"path/filepath"
"regexp"
"strings"
)
type Logs struct {
Timestamp string `json:"timestamp"`
Handler string `json:"handler"`
Level string `json:"level"`
Source string `json:"source"`
IpAddress string `json:"ipAddress"`
Message string `json:"message"`
}
var LogFile string = "./data/fail2ban.log"
var ParsedJson string = "./data/json_output.json"
var StatsByIPFile string = "./data/stats_by_ip.json"
func parseFile(stateFilePath string, logFilePath string, destinationDirectory string) {
// Init Parsed Log File Name
destinationFilePath := filepath.Join(destinationDirectory, "parsed.json")
// Load metadata
offset := checkState(stateFilePath).Offset
// Check if the log file has rolled over
file, err := os.Open(logFilePath)
if err != nil {
fmt.Printf("Error opening file: $%v", err)
return
}
defer file.Close()
stat, _ := file.Stat()
if stat.Size() < offset {
offset = 0
}
// Read out existing parsed log file
var logs []Logs
if data, err := os.ReadFile(destinationFilePath); err == nil && len(data) > 0 {
_ = json.Unmarshal(data, &logs)
}
// Define regex logic
const lenTimestamp = 23
dateRegex := regexp.MustCompile(`\d{4}-\d{2}-\d{2}`)
handlerRegex := regexp.MustCompile(`fail2ban\.\w+`)
ipRegex := regexp.MustCompile(`(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`)
levelRegex := regexp.MustCompile(`\s*(?:[A-Z]+)\s+`)
serviceRegex := regexp.MustCompile(`\s*(?:\[[a-z]+\])\s+`)
actionRegex := regexp.MustCompile(`(Found|already banned|Ban|Unban)`)
logEntry := Logs{}
_, err = file.Seek(offset, io.SeekStart)
if err != nil {
log.Fatalf("Error going to offset: %s\n", err)
}
// Parse the file and append to existing log files
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
if len(line) < lenTimestamp {
continue
} else if !dateRegex.MatchString(line[:lenTimestamp]) {
continue
}
timestamp := line[:lenTimestamp]; timestamp = strings.TrimSpace(timestamp)
logString := line[lenTimestamp:]
ipAddress := strings.TrimSpace(ipRegex.FindString(logString))
handler := strings.TrimSpace(handlerRegex.FindString(logString))
level := strings.TrimSpace(levelRegex.FindString(logString))
service := strings.TrimSpace(serviceRegex.FindString(logString))
action := strings.TrimSpace(actionRegex.FindString(logString))
logEntry.IpAddress = ipAddress
logEntry.Timestamp = timestamp
logEntry.Handler = handler
logEntry.Level = level
logEntry.Source = service
logEntry.Message = action
logs = append(logs, logEntry)
}
// Write parsed content and update metadata
jsonData, err := json.MarshalIndent(logs, "", " ")
_ = os.WriteFile(destinationFilePath, jsonData, 0644)
newOffset, _ := file.Seek(0, io.SeekCurrent)
newState := State{
Offset: newOffset,
}
updateState(stateFilePath, newState)
}
func parseLogsInJson() {
data, err := os.Open(LogFile)
if err != nil {
fmt.Printf("Error opening file: $%v", err)
return
}
defer data.Close()
logs := []Logs{}
const lenTimestamp = 23
dateRegex, _ := regexp.Compile(`\d{4}-\d{2}-\d{2}`)
handlerRegex, _ := regexp.Compile(`fail2ban\.\w+`)
ipRegex, _ := regexp.Compile(`(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`)
levelRegex, _ := regexp.Compile(`\s*(?:[A-Z]+)\s+`)
serviceRegex, _ := regexp.Compile(`\s*(?:\[[a-z]+\])\s+`)
actionRegex, _ := regexp.Compile(`(Found|already banned|Ban|Unban)`)
scanner := bufio.NewScanner(data)
logEntry := Logs{}
for scanner.Scan() {
line := scanner.Text()
if len(line) < lenTimestamp {
continue
} else if !dateRegex.MatchString(line[:lenTimestamp]) {
continue
}
timestamp := line[:lenTimestamp]; timestamp = strings.TrimSpace(timestamp)
logString := line[lenTimestamp:]
ipAddress := strings.TrimSpace(ipRegex.FindString(logString))
handler := strings.TrimSpace(handlerRegex.FindString(logString))
level := strings.TrimSpace(levelRegex.FindString(logString))
service := strings.TrimSpace(serviceRegex.FindString(logString))
action := strings.TrimSpace(actionRegex.FindString(logString))
logEntry.IpAddress = ipAddress
logEntry.Timestamp = timestamp
logEntry.Handler = handler
logEntry.Level = level
logEntry.Source = service
logEntry.Message = action
logs = append(logs, logEntry)
}
jsonData, err := json.MarshalIndent(logs, "", " ")
err = os.WriteFile(ParsedJson, jsonData, 0644)
}
+48
View File
@@ -0,0 +1,48 @@
package main
import (
"fmt"
"gonum.org/v1/plot"
"gonum.org/v1/plot/plotter"
"gonum.org/v1/plot/plotutil"
"gonum.org/v1/plot/vg"
)
func barChart(action string, variable string, counter map[string]float64) {
labels := []string{}
plottingCount := plotter.Values{}
for key, value := range counter {
if value > 100 {
labels = append(labels, key)
plottingCount = append(plottingCount, value)
}
}
p := plot.New()
p.Title.Text = fmt.Sprintf("%v by %v", action, variable)
p.Y.Label.Text = fmt.Sprintf("%v", action)
w := vg.Points(15)
barsA, err := plotter.NewBarChart(plottingCount, w)
if err != nil {
panic(err)
}
//barsA.LineStyle.Width = vg.Length(0)
barsA.Color = plotutil.Color(0)
barsA.Offset = 0
p.Add(barsA)
p.NominalX(labels...)
fileName := fmt.Sprintf("%v_%v_barchart.png", action, variable)
if err := p.Save(12*vg.Inch, 6*vg.Inch, fileName); err != nil {
panic(err)
}
}