diff --git a/README.md b/README.md index e69de29..551ed60 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,87 @@ +# Fail2Ban Log Parser + +## About + +This is a mini weekend project i did as one of my first steps in learning golang. The binary takes a f2b log file as input and a output directory as output, in which it creates a parsed.json and state.json file. Parsed.json contains the logs in a structured file and state.json is the metadata used to track at which point the parser last checked and if the file rolled over (log files tend to wipe after a certain amount of rows or create copies of themselfes while wiping their contents.) + +## Usage + +To build the project run + +```bash +make build +``` + +This will create a binary: `bin/parser` + +Once you run the binary you will get the usage menu: +``` +Usage: ./bin/parser --destDir= --source= + -d string + Destination Directory (shorthand) + -destDir string + Destination Directory + -s string + Source Log File (shorthand) + -source string + Source Log File +``` + +Lets say you have the following structure: + +``` +├── fail2ban/ +├── parser +└── raw-logs/ + └── fail2ban.log +``` + +You can run the parser: +``` +./parser -destDir=./fail2ban -source=./raw-logs/fail2ban.log +``` + +The parser will read the source file and create `parsed.json`and `state.json` in your destination directory. _*If destination directory does not exist, the parser will try to create it._ + +`parsed.json` will have the following structure: + +``` +[ + { + "timestamp": "2025-10-19 00:00:01,810", + "handler": "fail2ban.server", + "level": "INFO", + "source": "", + "ipAddress": "", + "message": "" + }, +... +] +``` + +You can import this in grafana or any other tool to analyse further. + +`state.json` will look like this: + +``` +{ + "offset": 3703746, + "size": 6079112 +} +``` + +Offset is the size in Bytes of how far the fail2ban.log file was read when the command was run the last time. + +Size is the tracker of the parsed log file size in Bytes. - I have implemented this for myself to have an overview, maybe one day I will write rollover logic based on this. + + +## More + +To read more about this project and how i made a Grafana dashboard you can read it on [My Blog](https://agres.online/blog/bruteforce) + + +## Developed with + +- make +- go 1.25.3 +- linux diff --git a/folder/parser b/folder/parser new file mode 100644 index 0000000..e69de29 diff --git a/makefile b/makefile index 360e5df..3aad4c6 100644 --- a/makefile +++ b/makefile @@ -1,5 +1,8 @@ run: go run ./src/. -buid: +build: go build -o bin/parser ./src/. + +clean: + rm bin/* diff --git a/src/analysis.go b/src/analysis.go index 32bb509..5902d51 100644 --- a/src/analysis.go +++ b/src/analysis.go @@ -7,12 +7,12 @@ import ( ) type StatsByIp struct { - IpAddress string `json:"ipAddress"` - TotalLogs int `json:"totalLogs"` - TotalFound int `json:"totalFound"` - TotalBanned int `json:"totalBanned"` - TotalUnbanned int `json:"totalUnbanned"` - Country string `json:"county"` + IpAddress string `json:"ipAddress"` + TotalLogs int `json:"totalLogs"` + TotalFound int `json:"totalFound"` + TotalBanned int `json:"totalBanned"` + TotalUnbanned int `json:"totalUnbanned"` + Country string `json:"county"` } func analyseLogs() { @@ -73,7 +73,6 @@ func analyseLogs() { } - func analyseExtractedData() { totalCounter := make(map[string]float64) diff --git a/src/main.go b/src/main.go index da70bae..93050e2 100644 --- a/src/main.go +++ b/src/main.go @@ -8,11 +8,6 @@ import ( type function func() func main() { - - //timedRun(parseLogsInJson) - //timedRun(analyseLogs) - //timedRun(analyseExtractedData) - //timedRun(starter) starter() } diff --git a/src/parser.go b/src/parser.go index 31805d5..f7932e2 100644 --- a/src/parser.go +++ b/src/parser.go @@ -9,7 +9,8 @@ import ( ) type State struct { - Offset int64 `json:"offset"` + Offset int64 `json:"offset"` + ParsedFileSize int64 `json:"size"` } func starter() { @@ -20,7 +21,7 @@ func starter() { flag.StringVar(source, "s", "", "Source Log File (shorthand)") flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s --destDir --source \n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s --destDir= --source=\n", os.Args[0]) flag.PrintDefaults() } @@ -31,15 +32,16 @@ func starter() { os.Exit(1) } - checkParameters(*destinationDirectory, *source) + stateFile := checkParameters(*destinationDirectory, *source) + + parseFile(*stateFile, *source, *destinationDirectory) } /* * TODO: This function does not check read/write permissions yet */ -func checkParameters(destinationDirectory string, source string) { - +func checkParameters(destinationDirectory string, source string) (stateFilePath *string) { if _, err := os.Stat(source); os.IsNotExist(err) { fmt.Fprintf(os.Stderr, "Error: source file does not exist: %s\n", source) @@ -66,7 +68,7 @@ func checkParameters(destinationDirectory string, source string) { os.Exit(1) } - parseFile(stateFile, source, destinationDirectory) + return &stateFile } func initState(destinationDirectory string) (stateFilePath string, err error) { @@ -81,6 +83,7 @@ func initState(destinationDirectory string) (stateFilePath string, err error) { state := State{ Offset: 0, + ParsedFileSize: 0, } data, err := json.MarshalIndent(state, "", " ") if err != nil { diff --git a/src/parser_f2b.go b/src/parser_f2b.go index e195ac4..b1b5664 100644 --- a/src/parser_f2b.go +++ b/src/parser_f2b.go @@ -32,7 +32,8 @@ func parseFile(stateFilePath string, logFilePath string, destinationDirectory st destinationFilePath := filepath.Join(destinationDirectory, "parsed.json") // Load metadata - offset := checkState(stateFilePath).Offset + metadata := checkState(stateFilePath) + offset := metadata.Offset // Check if the log file has rolled over file, err := os.Open(logFilePath) @@ -105,63 +106,20 @@ func parseFile(stateFilePath string, logFilePath string, destinationDirectory st jsonData, err := json.MarshalIndent(logs, "", " ") _ = os.WriteFile(destinationFilePath, jsonData, 0644) - newOffset, _ := file.Seek(0, io.SeekCurrent) - newState := State{ - Offset: newOffset, - } - updateState(stateFilePath, newState) -} - -func parseLogsInJson() { - - data, err := os.Open(LogFile) + // Get parsed log file size + logFile, err := os.Open(destinationFilePath) if err != nil { fmt.Printf("Error opening file: $%v", err) return } - defer data.Close() + defer logFile.Close() + stat, _ = logFile.Stat() + parsedLogfileSize := stat.Size() - logs := []Logs{} - - const lenTimestamp = 23 - dateRegex, _ := regexp.Compile(`\d{4}-\d{2}-\d{2}`) - handlerRegex, _ := regexp.Compile(`fail2ban\.\w+`) - ipRegex, _ := regexp.Compile(`(\b25[0-5]|\b2[0-4][0-9]|\b[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) - levelRegex, _ := regexp.Compile(`\s*(?:[A-Z]+)\s+`) - serviceRegex, _ := regexp.Compile(`\s*(?:\[[a-z]+\])\s+`) - actionRegex, _ := regexp.Compile(`(Found|already banned|Ban|Unban)`) - - scanner := bufio.NewScanner(data) - logEntry := Logs{} - - for scanner.Scan() { - line := scanner.Text() - - if len(line) < lenTimestamp { - continue - } else if !dateRegex.MatchString(line[:lenTimestamp]) { - continue - } - - timestamp := line[:lenTimestamp]; timestamp = strings.TrimSpace(timestamp) - logString := line[lenTimestamp:] - - ipAddress := strings.TrimSpace(ipRegex.FindString(logString)) - handler := strings.TrimSpace(handlerRegex.FindString(logString)) - level := strings.TrimSpace(levelRegex.FindString(logString)) - service := strings.TrimSpace(serviceRegex.FindString(logString)) - action := strings.TrimSpace(actionRegex.FindString(logString)) - - logEntry.IpAddress = ipAddress - logEntry.Timestamp = timestamp - logEntry.Handler = handler - logEntry.Level = level - logEntry.Source = service - logEntry.Message = action - - logs = append(logs, logEntry) + newOffset, _ := file.Seek(0, io.SeekCurrent) + newState := State{ + Offset: newOffset, + ParsedFileSize: parsedLogfileSize, } - - jsonData, err := json.MarshalIndent(logs, "", " ") - err = os.WriteFile(ParsedJson, jsonData, 0644) + updateState(stateFilePath, newState) }