mirror of
https://github.com/agresdominik/f2b_parser.git
synced 2026-04-21 18:05:47 +00:00
code
This commit is contained in:
+124
@@ -0,0 +1,124 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
type StatsByIp struct {
|
||||
IpAddress string `json:"ipAddress"`
|
||||
TotalLogs int `json:"totalLogs"`
|
||||
TotalFound int `json:"totalFound"`
|
||||
TotalBanned int `json:"totalBanned"`
|
||||
TotalUnbanned int `json:"totalUnbanned"`
|
||||
Country string `json:"county"`
|
||||
}
|
||||
|
||||
func analyseLogs() {
|
||||
|
||||
data, err := os.ReadFile(ParsedJson)
|
||||
if err != nil {
|
||||
fmt.Printf("Error opening file: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
statsByIp := make(map[string]*StatsByIp)
|
||||
var fileData []Logs
|
||||
err = json.Unmarshal(data, &fileData)
|
||||
if err != nil {
|
||||
fmt.Printf("Error unmarshaling data: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range fileData {
|
||||
|
||||
ip := entry.IpAddress
|
||||
if ip == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
action := entry.Message
|
||||
|
||||
|
||||
if _, exists := statsByIp[ip]; !exists {
|
||||
country, err := getIpAddressCountry(ip)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed getting ip-address with error: %v", err)
|
||||
}
|
||||
statsByIp[ip] = &StatsByIp{
|
||||
IpAddress: ip,
|
||||
Country: country,
|
||||
}
|
||||
}
|
||||
|
||||
statsByIp[ip].TotalLogs += 1
|
||||
|
||||
switch action {
|
||||
case "Found":
|
||||
statsByIp[ip].TotalFound += 1
|
||||
case "Ban", "already banned":
|
||||
statsByIp[ip].TotalBanned += 1
|
||||
case "Unban":
|
||||
statsByIp[ip].TotalUnbanned += 1
|
||||
}
|
||||
}
|
||||
|
||||
jsonData, err := json.MarshalIndent(statsByIp, "", " ")
|
||||
if err != nil {
|
||||
fmt.Println("Error marshalling the stats file.")
|
||||
return
|
||||
}
|
||||
err = os.WriteFile(StatsByIPFile, jsonData, 0644)
|
||||
|
||||
}
|
||||
|
||||
|
||||
func analyseExtractedData() {
|
||||
|
||||
totalCounter := make(map[string]float64)
|
||||
totalBans := make(map[string]float64)
|
||||
totalConnections := make(map[string]float64)
|
||||
|
||||
data, err := os.ReadFile(StatsByIPFile)
|
||||
if err != nil {
|
||||
fmt.Printf("Error opening file: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
var statsByIp map[string]StatsByIp
|
||||
err = json.Unmarshal(data, &statsByIp)
|
||||
if err != nil {
|
||||
fmt.Printf("Error unmarshaling data: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range statsByIp {
|
||||
country := entry.Country
|
||||
|
||||
if _, exists := totalCounter[country]; !exists {
|
||||
totalCounter[country] = 0
|
||||
} else {
|
||||
totalCounter[country] += 1
|
||||
}
|
||||
if entry.TotalBanned > 0 {
|
||||
if _, exists := totalBans[country]; !exists {
|
||||
totalBans[country] = float64(entry.TotalBanned)
|
||||
} else {
|
||||
totalBans[country] += float64(entry.TotalBanned)
|
||||
}
|
||||
}
|
||||
if entry.TotalFound > 0 {
|
||||
if _, exists := totalConnections[country]; !exists {
|
||||
totalConnections[country] = float64(entry.TotalFound)
|
||||
} else {
|
||||
totalConnections[country] += float64(entry.TotalFound)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
barChart("Individual IPs", "Country", totalCounter)
|
||||
barChart("Total Banned Ips", "Country", totalBans)
|
||||
barChart("Total found connections", "Country", totalConnections)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type OkResponseJson struct {
|
||||
Ipaddress string `json:"ipaddress"`
|
||||
Continent_code string `json:"continent_code"`
|
||||
Continent_name string `json:"continent_name"`
|
||||
Country_code string `json:"country_code"`
|
||||
Country_name string `json:"country_name"`
|
||||
}
|
||||
|
||||
func getIpAddressCountry(ipAddress string) (country string, err error) {
|
||||
|
||||
url := fmt.Sprintf("https://api.ipaddress.com/iptocountry?format=json&ip=%s", ipAddress)
|
||||
request, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Failed creating request: %v", err)
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
response , err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Failed making request: %v", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Failed reading body: %s", err)
|
||||
}
|
||||
|
||||
var responseFormat OkResponseJson
|
||||
err = json.Unmarshal(body, &responseFormat)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Failed parsing response body with error: %v", err)
|
||||
}
|
||||
|
||||
return responseFormat.Country_name, nil
|
||||
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type function func()
|
||||
|
||||
func main() {
|
||||
|
||||
//timedRun(parseLogsInJson)
|
||||
//timedRun(analyseLogs)
|
||||
//timedRun(analyseExtractedData)
|
||||
timedRun(starter)
|
||||
}
|
||||
|
||||
func timedRun(fn function) {
|
||||
|
||||
now := time.Now().UnixMilli()
|
||||
fn()
|
||||
after := time.Now().UnixMilli()
|
||||
|
||||
runtime := after - now
|
||||
fmt.Printf("\nTotal runtime: %v", runtime)
|
||||
}
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type State struct {
|
||||
Offset int64 `json:"offset"`
|
||||
}
|
||||
|
||||
func starter() {
|
||||
|
||||
destinationDirectory := flag.String("destDir", "", "Destination Directory")
|
||||
flag.StringVar(destinationDirectory, "d", "", "Destination Directory (shorthand)")
|
||||
source := flag.String("source", "", "Source Log File")
|
||||
flag.StringVar(source, "s", "", "Source Log File (shorthand)")
|
||||
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "Usage: %s --destDir <dir> --source <file>\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
|
||||
flag.Parse()
|
||||
|
||||
if *destinationDirectory == "" || *source == "" {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
checkParameters(*destinationDirectory, *source)
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* TODO: This function does not check read/write permissions yet
|
||||
*/
|
||||
func checkParameters(destinationDirectory string, source string) {
|
||||
|
||||
|
||||
if _, err := os.Stat(source); os.IsNotExist(err) {
|
||||
fmt.Fprintf(os.Stderr, "Error: source file does not exist: %s\n", source)
|
||||
os.Exit(1)
|
||||
} else if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed reading source file: %s with error: %s\n", source, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(destinationDirectory); os.IsNotExist(err){
|
||||
err = os.MkdirAll(destinationDirectory, 0755)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed creating directory file: %s with error: %s\n", source, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
} else if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed reading directory file: %s with error: %s\nDid not try to create.", source, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
stateFile, err := initState(destinationDirectory)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: failed initialising state file: %s\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
parseFile(stateFile, source, destinationDirectory)
|
||||
}
|
||||
|
||||
func initState(destinationDirectory string) (stateFilePath string, err error) {
|
||||
|
||||
stateFile := filepath.Join(destinationDirectory, "state.json")
|
||||
|
||||
if _, err := os.Stat(stateFile); err == nil {
|
||||
return stateFile, nil
|
||||
} else if os.IsNotExist(err) {
|
||||
fmt.Println("No state file found, creating...")
|
||||
}
|
||||
|
||||
state := State{
|
||||
Offset: 0,
|
||||
}
|
||||
data, err := json.MarshalIndent(state, "", " ")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
err = os.WriteFile(stateFile, data, 0644)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
fmt.Printf("State file initialised: %s\n", stateFile)
|
||||
return stateFile, nil
|
||||
}
|
||||
|
||||
func checkState(stateFile string) State {
|
||||
|
||||
data, err := os.ReadFile(stateFile)
|
||||
if err != nil {
|
||||
fmt.Printf("Error opening file: %v", err)
|
||||
return State{}
|
||||
}
|
||||
|
||||
var state State
|
||||
|
||||
err = json.Unmarshal(data, &state)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error reading state file: %s", err)
|
||||
return State{}
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func updateState(stateFile string, newState State) error {
|
||||
|
||||
data, err := json.MarshalIndent(newState, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.WriteFile(stateFile, data, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Logs struct {
|
||||
Timestamp string `json:"timestamp"`
|
||||
Handler string `json:"handler"`
|
||||
Level string `json:"level"`
|
||||
Source string `json:"source"`
|
||||
IpAddress string `json:"ipAddress"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
var LogFile string = "./data/fail2ban.log"
|
||||
var ParsedJson string = "./data/json_output.json"
|
||||
var StatsByIPFile string = "./data/stats_by_ip.json"
|
||||
|
||||
|
||||
func parseFile(stateFilePath string, logFilePath string, destinationDirectory string) {
|
||||
|
||||
// Init Parsed Log File Name
|
||||
destinationFilePath := filepath.Join(destinationDirectory, "parsed.json")
|
||||
|
||||
// Load metadata
|
||||
offset := checkState(stateFilePath).Offset
|
||||
|
||||
// Check if the log file has rolled over
|
||||
file, err := os.Open(logFilePath)
|
||||
if err != nil {
|
||||
fmt.Printf("Error opening file: $%v", err)
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stat, _ := file.Stat()
|
||||
if stat.Size() < offset {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
// Read out existing parsed log file
|
||||
var logs []Logs
|
||||
if data, err := os.ReadFile(destinationFilePath); err == nil && len(data) > 0 {
|
||||
_ = json.Unmarshal(data, &logs)
|
||||
}
|
||||
|
||||
// Define regex logic
|
||||
const lenTimestamp = 23
|
||||
dateRegex := regexp.MustCompile(`\d{4}-\d{2}-\d{2}`)
|
||||
handlerRegex := regexp.MustCompile(`fail2ban\.\w+`)
|
||||
ipRegex := regexp.MustCompile(`(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`)
|
||||
levelRegex := regexp.MustCompile(`\s*(?:[A-Z]+)\s+`)
|
||||
serviceRegex := regexp.MustCompile(`\s*(?:\[[a-z]+\])\s+`)
|
||||
actionRegex := regexp.MustCompile(`(Found|already banned|Ban|Unban)`)
|
||||
|
||||
logEntry := Logs{}
|
||||
|
||||
|
||||
_, err = file.Seek(offset, io.SeekStart)
|
||||
if err != nil {
|
||||
log.Fatalf("Error going to offset: %s\n", err)
|
||||
}
|
||||
|
||||
// Parse the file and append to existing log files
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
|
||||
line := scanner.Text()
|
||||
|
||||
if len(line) < lenTimestamp {
|
||||
continue
|
||||
} else if !dateRegex.MatchString(line[:lenTimestamp]) {
|
||||
continue
|
||||
}
|
||||
|
||||
timestamp := line[:lenTimestamp]; timestamp = strings.TrimSpace(timestamp)
|
||||
logString := line[lenTimestamp:]
|
||||
|
||||
ipAddress := strings.TrimSpace(ipRegex.FindString(logString))
|
||||
handler := strings.TrimSpace(handlerRegex.FindString(logString))
|
||||
level := strings.TrimSpace(levelRegex.FindString(logString))
|
||||
service := strings.TrimSpace(serviceRegex.FindString(logString))
|
||||
action := strings.TrimSpace(actionRegex.FindString(logString))
|
||||
|
||||
logEntry.IpAddress = ipAddress
|
||||
logEntry.Timestamp = timestamp
|
||||
logEntry.Handler = handler
|
||||
logEntry.Level = level
|
||||
logEntry.Source = service
|
||||
logEntry.Message = action
|
||||
|
||||
logs = append(logs, logEntry)
|
||||
}
|
||||
|
||||
// Write parsed content and update metadata
|
||||
jsonData, err := json.MarshalIndent(logs, "", " ")
|
||||
_ = os.WriteFile(destinationFilePath, jsonData, 0644)
|
||||
|
||||
newOffset, _ := file.Seek(0, io.SeekCurrent)
|
||||
newState := State{
|
||||
Offset: newOffset,
|
||||
}
|
||||
updateState(stateFilePath, newState)
|
||||
}
|
||||
|
||||
func parseLogsInJson() {
|
||||
|
||||
data, err := os.Open(LogFile)
|
||||
if err != nil {
|
||||
fmt.Printf("Error opening file: $%v", err)
|
||||
return
|
||||
}
|
||||
defer data.Close()
|
||||
|
||||
logs := []Logs{}
|
||||
|
||||
const lenTimestamp = 23
|
||||
dateRegex, _ := regexp.Compile(`\d{4}-\d{2}-\d{2}`)
|
||||
handlerRegex, _ := regexp.Compile(`fail2ban\.\w+`)
|
||||
ipRegex, _ := regexp.Compile(`(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`)
|
||||
levelRegex, _ := regexp.Compile(`\s*(?:[A-Z]+)\s+`)
|
||||
serviceRegex, _ := regexp.Compile(`\s*(?:\[[a-z]+\])\s+`)
|
||||
actionRegex, _ := regexp.Compile(`(Found|already banned|Ban|Unban)`)
|
||||
|
||||
scanner := bufio.NewScanner(data)
|
||||
logEntry := Logs{}
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
if len(line) < lenTimestamp {
|
||||
continue
|
||||
} else if !dateRegex.MatchString(line[:lenTimestamp]) {
|
||||
continue
|
||||
}
|
||||
|
||||
timestamp := line[:lenTimestamp]; timestamp = strings.TrimSpace(timestamp)
|
||||
logString := line[lenTimestamp:]
|
||||
|
||||
ipAddress := strings.TrimSpace(ipRegex.FindString(logString))
|
||||
handler := strings.TrimSpace(handlerRegex.FindString(logString))
|
||||
level := strings.TrimSpace(levelRegex.FindString(logString))
|
||||
service := strings.TrimSpace(serviceRegex.FindString(logString))
|
||||
action := strings.TrimSpace(actionRegex.FindString(logString))
|
||||
|
||||
logEntry.IpAddress = ipAddress
|
||||
logEntry.Timestamp = timestamp
|
||||
logEntry.Handler = handler
|
||||
logEntry.Level = level
|
||||
logEntry.Source = service
|
||||
logEntry.Message = action
|
||||
|
||||
logs = append(logs, logEntry)
|
||||
}
|
||||
|
||||
jsonData, err := json.MarshalIndent(logs, "", " ")
|
||||
err = os.WriteFile(ParsedJson, jsonData, 0644)
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gonum.org/v1/plot"
|
||||
"gonum.org/v1/plot/plotter"
|
||||
"gonum.org/v1/plot/plotutil"
|
||||
"gonum.org/v1/plot/vg"
|
||||
)
|
||||
|
||||
|
||||
func barChart(action string, variable string, counter map[string]float64) {
|
||||
|
||||
labels := []string{}
|
||||
|
||||
plottingCount := plotter.Values{}
|
||||
for key, value := range counter {
|
||||
if value > 100 {
|
||||
labels = append(labels, key)
|
||||
plottingCount = append(plottingCount, value)
|
||||
}
|
||||
}
|
||||
|
||||
p := plot.New()
|
||||
|
||||
p.Title.Text = fmt.Sprintf("%v by %v", action, variable)
|
||||
p.Y.Label.Text = fmt.Sprintf("%v", action)
|
||||
|
||||
w := vg.Points(15)
|
||||
|
||||
barsA, err := plotter.NewBarChart(plottingCount, w)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
//barsA.LineStyle.Width = vg.Length(0)
|
||||
barsA.Color = plotutil.Color(0)
|
||||
barsA.Offset = 0
|
||||
|
||||
p.Add(barsA)
|
||||
p.NominalX(labels...)
|
||||
|
||||
fileName := fmt.Sprintf("%v_%v_barchart.png", action, variable)
|
||||
|
||||
if err := p.Save(12*vg.Inch, 6*vg.Inch, fileName); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user