agresdominik/f2b_parser
1
0
Fork 0
mirror of https://github.com/agresdominik/f2b_parser.git synced 2026-04-21 10:01:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

added file size in metadata

Browse Source
This commit is contained in:
Dominik
2025-11-04 12:11:54 +01:00
parent d018b9bf1f
commit 321422d1d7
7 changed files with 118 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+87
View File
@@ -0,0 +1,87 @@
# Fail2Ban Log Parser
## About
This is a mini weekend project i did as one of my first steps in learning golang. The binary takes a f2b log file as input and a output directory as output, in which it creates a parsed.json and state.json file. Parsed.json contains the logs in a structured file and state.json is the metadata used to track at which point the parser last checked and if the file rolled over (log files tend to wipe after a certain amount of rows or create copies of themselfes while wiping their contents.)
## Usage
To build the project run
```bash
make build
```
This will create a binary: `bin/parser`
Once you run the binary you will get the usage menu:
```
Usage: ./bin/parser --destDir=<dir> --source=<file>
-d string
Destination Directory (shorthand)
-destDir string
Destination Directory
-s string
Source Log File (shorthand)
-source string
Source Log File
```
Lets say you have the following structure:
```
├── fail2ban/
├── parser
└── raw-logs/
└── fail2ban.log
```
You can run the parser:
```
./parser -destDir=./fail2ban -source=./raw-logs/fail2ban.log
```
The parser will read the source file and create `parsed.json`and `state.json` in your destination directory. _*If destination directory does not exist, the parser will try to create it._
`parsed.json` will have the following structure:
```
[
{
"timestamp": "2025-10-19 00:00:01,810",
"handler": "fail2ban.server",
"level": "INFO",
"source": "",
"ipAddress": "",
"message": ""
},
...
]
```
You can import this in grafana or any other tool to analyse further.
`state.json` will look like this:
```
{
"offset": 3703746,
"size": 6079112
}
```
Offset is the size in Bytes of how far the fail2ban.log file was read when the command was run the last time.
Size is the tracker of the parsed log file size in Bytes. - I have implemented this for myself to have an overview, maybe one day I will write rollover logic based on this.
## More
To read more about this project and how i made a Grafana dashboard you can read it on [My Blog](https://agres.online/blog/bruteforce)
## Developed with
- make
- go 1.25.3
- linux
folder/parser
View File
makefile
+4 -1
View File
@@ -1,5 +1,8 @@
run:
go run ./src/.
buid:
build:
go build -o bin/parser ./src/.
clean:
rm bin/*
src/analysis.go
-1
View File
@@ -73,7 +73,6 @@ func analyseLogs() {
}
func analyseExtractedData() {
totalCounter := make(map[string]float64)
src/main.go
-5
View File
@@ -8,11 +8,6 @@ import (
type function func()
func main() {
//timedRun(parseLogsInJson)
//timedRun(analyseLogs)
//timedRun(analyseExtractedData)
//timedRun(starter)
starter()
}
src/parser.go
+8 -5
View File
@@ -10,6 +10,7 @@ import (
type State struct {
Offset int64 `json:"offset"`
ParsedFileSize int64 `json:"size"`
}
func starter() {
@@ -20,7 +21,7 @@ func starter() {
flag.StringVar(source, "s", "", "Source Log File (shorthand)")
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s --destDir <dir> --source <file>\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Usage: %s --destDir=<dir> --source=<file>\n", os.Args[0])
flag.PrintDefaults()
}
@@ -31,15 +32,16 @@ func starter() {
os.Exit(1)
}
checkParameters(*destinationDirectory, *source)
stateFile := checkParameters(*destinationDirectory, *source)
parseFile(*stateFile, *source, *destinationDirectory)
}
/*
* TODO: This function does not check read/write permissions yet
*/
func checkParameters(destinationDirectory string, source string) {
func checkParameters(destinationDirectory string, source string) (stateFilePath *string) {
if _, err := os.Stat(source); os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Error: source file does not exist: %s\n", source)
@@ -66,7 +68,7 @@ func checkParameters(destinationDirectory string, source string) {
os.Exit(1)
}
parseFile(stateFile, source, destinationDirectory)
return &stateFile
}
func initState(destinationDirectory string) (stateFilePath string, err error) {
@@ -81,6 +83,7 @@ func initState(destinationDirectory string) (stateFilePath string, err error) {
state := State{
Offset: 0,
ParsedFileSize: 0,
}
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
src/parser_f2b.go
+12 -54
View File
@@ -32,7 +32,8 @@ func parseFile(stateFilePath string, logFilePath string, destinationDirectory st
destinationFilePath := filepath.Join(destinationDirectory, "parsed.json")
// Load metadata
offset := checkState(stateFilePath).Offset
metadata := checkState(stateFilePath)
offset := metadata.Offset
// Check if the log file has rolled over
file, err := os.Open(logFilePath)
@@ -105,63 +106,20 @@ func parseFile(stateFilePath string, logFilePath string, destinationDirectory st
jsonData, err := json.MarshalIndent(logs, "", " ")
_ = os.WriteFile(destinationFilePath, jsonData, 0644)
newOffset, _ := file.Seek(0, io.SeekCurrent)
newState := State{
Offset: newOffset,
}
updateState(stateFilePath, newState)
}
func parseLogsInJson() {
data, err := os.Open(LogFile)
// Get parsed log file size
logFile, err := os.Open(destinationFilePath)
if err != nil {
fmt.Printf("Error opening file: $%v", err)
return
}
defer data.Close()
defer logFile.Close()
stat, _ = logFile.Stat()
parsedLogfileSize := stat.Size()
logs := []Logs{}
const lenTimestamp = 23
dateRegex, _ := regexp.Compile(`\d{4}-\d{2}-\d{2}`)
handlerRegex, _ := regexp.Compile(`fail2ban\.\w+`)
ipRegex, _ := regexp.Compile(`(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`)
levelRegex, _ := regexp.Compile(`\s*(?:[A-Z]+)\s+`)
serviceRegex, _ := regexp.Compile(`\s*(?:\[[a-z]+\])\s+`)
actionRegex, _ := regexp.Compile(`(Found|already banned|Ban|Unban)`)
scanner := bufio.NewScanner(data)
logEntry := Logs{}
for scanner.Scan() {
line := scanner.Text()
if len(line) < lenTimestamp {
continue
} else if !dateRegex.MatchString(line[:lenTimestamp]) {
continue
newOffset, _ := file.Seek(0, io.SeekCurrent)
newState := State{
Offset: newOffset,
ParsedFileSize: parsedLogfileSize,
}
timestamp := line[:lenTimestamp]; timestamp = strings.TrimSpace(timestamp)
logString := line[lenTimestamp:]
ipAddress := strings.TrimSpace(ipRegex.FindString(logString))
handler := strings.TrimSpace(handlerRegex.FindString(logString))
level := strings.TrimSpace(levelRegex.FindString(logString))
service := strings.TrimSpace(serviceRegex.FindString(logString))
action := strings.TrimSpace(actionRegex.FindString(logString))
logEntry.IpAddress = ipAddress
logEntry.Timestamp = timestamp
logEntry.Handler = handler
logEntry.Level = level
logEntry.Source = service
logEntry.Message = action
logs = append(logs, logEntry)
}
jsonData, err := json.MarshalIndent(logs, "", " ")
err = os.WriteFile(ParsedJson, jsonData, 0644)
updateState(stateFilePath, newState)
}