Merge pull request #18 from klouddb/release-v2.3

PostgreSQL 17 Support (Commit: 87a6f0f)
postgres/check.go: Added new checks for PostgreSQL 17 across multiple categories
postgres/installation/installation.go: Added systemd service verification for PostgreSQL 17
SSL Handling and Structure Improvements (Commit: 830c261)
Configuration: Updated config handling for SSL and ping check flags
Database Connections: Enhanced PostgreSQL and MySQL connection handling
Log Parser: Improved log parsing with better SSL handling
Templates: Updated HTML report templates
Integration Tests: Enhanced test case structure

Author: princeparmar

.gitignore

@@ -49,4 +49,6 @@ detailed_output.csv
 all_checks.json
 
 postgresql.conf
-testoutput.log
+testoutput.log
+
+*.prof

README.md

@@ -100,6 +100,11 @@ port="5432"
 user="postgres"
 dbname="postgres"
 password="xxxxx" 
+sslmode="require"
+sslcert="path/to/cert"
+sslkey="path/to/key"
+sslrootcert="path/to/rootcert"
+pingCheck=true
 maxIdleConn = 2
 maxOpenConn = 2
 

cmd/ciscollector/logparserrunner.go

@@ -112,7 +112,7 @@ func runLogParserWithMultipleParser(ctx context.Context, runCmd bool, logParserC
 		logparser.PrintTerminalResultsForLogParser(ctx, allParser, outputType)
 	}
 
-	htmlReportHelper.RenderLogparserResponse(ctx, store, allParser)
+	htmlReportHelper.RenderLogparserResponse(ctx, allParser)
 	return nil
 }
 

docker_testing/integrationtest/testcase.go

@@ -188,7 +188,7 @@ func testUnusedHbaLines(prefix, file string) {
 	}
 
 	out := buf.String()
-	if strings.Contains(out, "In logline prefix, please set '%u' and '%d'") || strings.Contains(out, "Please set log_line_prefix to '%h' or '%r' or enable log_connections") {
+	if strings.Contains(out, "In logline prefix, please set '%u' and '%d'") || strings.Contains(out, "please set log_line_prefix") {
 		fmt.Println("skipping test for unused files as required details are not available in prefix:", prefix)
 		return
 	}
@@ -199,7 +199,7 @@ func testUnusedHbaLines(prefix, file string) {
 		os.Exit(1)
 	}
 
-	if strings.Contains(out, `Unused lines found from given log file: [11 23 28]`) {
+	if strings.Contains(out, `Unused lines found from given log file: [11 23 28]`) || strings.Contains(out, `Unused lines found from given log file: [11 16 17 23 28]`) {
 		fmt.Println("unused lines test is working fine for prefix:", prefix)
 		return
 	}

htmlreport/logparser_helper.go

@@ -2,7 +2,6 @@ package htmlreport
 
 import (
 	"context"
-	"database/sql"
 	"strings"
 
 	"github.com/klouddb/klouddbshield/pkg/hbarules"
@@ -12,7 +11,7 @@ import (
 )
 
 type LogparserHTMLReport struct {
-	Error           string
+	Error           []string
 	InactiveUsers   *SimplifiedInactiveUserData
 	UniqueIPs       *UniqueIPRenderData
 	UnusedHBALines  *UnusedHBALinesRenderData
@@ -61,11 +60,11 @@ func GetSimplifiedInactiveUsers(userdata [][]string) *SimplifiedInactiveUserData
 
 func (h *HtmlReportHelper) RanderLogParserError(err error) {
 	h.AddTab("Log Parser", LogparserHTMLReport{
-		Error: err.Error(),
+		Error: []string{err.Error()},
 	})
 }
 
-func (h *HtmlReportHelper) RenderLogparserResponse(ctx context.Context, store *sql.DB, parsers []runner.Parser) {
+func (h *HtmlReportHelper) RenderLogparserResponse(ctx context.Context, parsers []runner.Parser) {
 	data := LogparserHTMLReport{}
 
 	for _, r := range parsers {
@@ -92,6 +91,8 @@ func (h *HtmlReportHelper) RenderLogparserResponse(ctx context.Context, store *s
 			data.SQLInjection = &SQLInjectionRenderData{
 				Logs: r.GetResult(ctx),
 			}
+		case *logparser.ErrorHelper:
+			data.Error = append(data.Error, r.Error())
 		}
 	}
 

htmlreport/template/logparser.tmpl

@@ -1,9 +1,11 @@
 {{ define "logparserbody" }}
     <div class="wrapper">
         <div class="myContainer">
-            {{ if .Error }}
-                <h3> Becuase of some error we were not able to generate the report for Log Parser.</h3>
-                <p>{{ .Error }}</p>
+            {{ if and (not (eq .Error nil)) (gt (len .Error) 0) }}
+                <h3>There are some errors while parsing the log file. Please check the following:</h3>
+                {{ range .Error }}
+                    <p style="color: #a72727;">{{ . }}</p>
+                {{ end }}
             {{ end }}
             {{ if .InactiveUsers }}
                 <div class="data-container">

kshieldconfig_example.toml

@@ -5,7 +5,14 @@
 # port = "5432"
 # user = "postgres"
 # password = "password123"
+# sslmode = "disable"
+# sslcert="path/to/cert"
+# sslkey="path/to/key"
+# sslrootcert="path/to/rootcert"
+# pingCheck = true
 # dbname = "mydb"
+# sslmode = "disable"
+# pingCheck = true
 # maxIdleConn = 10
 # maxOpenConn = 100
 
@@ -14,6 +21,7 @@
 # port="3306"
 # user="root"
 # password="mysql111"
+# pingCheck = true
 # maxIdleConn = 2
 # maxOpenConn = 2
 

pkg/config/config.go

@@ -217,8 +217,9 @@ type MySQL struct {
 	Password string `toml:"password"`
 	// DBName      string `toml:"dbname"`
 	// SSLmode     string `toml:"sslmode"`
-	MaxIdleConn int `toml:"maxIdleConn"`
-	MaxOpenConn int `toml:"maxOpenConn"`
+	PingCheck   bool `toml:"pingCheck"`
+	MaxIdleConn int  `toml:"maxIdleConn"`
+	MaxOpenConn int  `toml:"maxOpenConn"`
 }
 
 func (p *MySQL) HtmlReportName() string {
@@ -858,6 +859,12 @@ func LoadConfig(configPath string) (*Config, error) {
 		return c, fmt.Errorf("unmarshal: %v", err)
 	}
 
+	if c.Postgres != nil {
+		if c.Postgres.SSLmode == "" {
+			c.Postgres.SSLmode = "disable"
+		}
+	}
+
 	return c, nil
 }
 

pkg/logparser/hba_unused_lines.go

@@ -24,12 +24,21 @@ func NewUnusedHBALineHelper(store *sql.DB) *UnusedHBALineHelper {
 
 func (i *UnusedHBALineHelper) Init(ctx context.Context, logParserCnf *config.LogParser) error {
 	// check if postgres setting contains required variable or connection logs
-	if !strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%h") && !strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%r") {
-		return fmt.Errorf("Please set log_line_prefix to '%%h' or '%%r' or enable log_connections")
-	}
+	// for unused hba parsing we need to have any 2 of the following
+	// 1. log_line_prefix contains %h or %r
+	// 2. log_connections is enabled
+	// 3. log_line_prefix contains %u and %d
 
-	if !strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%u") || !strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%d") {
-		return fmt.Errorf("In logline prefix, please set '%s' and '%s'\n", "%u", "%d") // using printf to avoid the warning for %d in println
+	if logParserCnf.PgSettings.LogConnections {
+		if !(strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%h") || strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%r")) &&
+			!(strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%u") && strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%d")) {
+			return fmt.Errorf("with log_connections enabled, please set log_line_prefix to '%%h' or '%%r' or '%%u' and '%%d'")
+		}
+	} else {
+		if !(strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%h") || strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%r")) ||
+			!(strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%u") && strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%d")) {
+			return fmt.Errorf("please set log_line_prefix to '%%h' or '%%r' or '%%u' and '%%d'")
+		}
 	}
 
 	var hbaRules []model.HBAFIleRules

pkg/logparser/inactive_users.go

@@ -26,7 +26,7 @@ func NewInactiveUsersHelper(store *sql.DB) *InactiveUsersHelper {
 func (i *InactiveUsersHelper) Init(ctx context.Context, logParserCnf *config.LogParser) error {
 	// check if postgres setting contains required variable or connection logs
 	if !strings.Contains(logParserCnf.PgSettings.LogLinePrefix, "%u") && !logParserCnf.PgSettings.LogConnections {
-		return fmt.Errorf("Please set log_line_prefix to '%%u' or enable log_connections")
+		return fmt.Errorf("please set log_line_prefix to '%%u' or enable log_connections")
 	}
 
 	i.UniqueUserParser = parselog.NewUserParser(logParserCnf)