Merge branch 'release/v0.5.0-alpha'

Author: andreaskoch

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v0.5.0-alpha] - 2020-11-16
+
+### Changed
+- Ignore invalid SSL certificates
+
+### Added
+- Log response headers
+
 ## [v0.4.1-alpha] - 2020-11-16
 
 ### Fixed

README.md

@@ -64,21 +64,21 @@ You can download binaries for Linux, macOS and Windows from [github.com »andrea
 Linux:
 
 ```bash
-curl -L https://github.com/andreaskoch/gargantua/releases/download/v0.4.1-alpha/gargantua_linux_amd64 -o gargantua
+curl -L https://github.com/andreaskoch/gargantua/releases/download/v0.5.0-alpha/gargantua_linux_amd64 -o gargantua
 chmod +x gargantua
 ```
 
 macOS:
 
 ```bash
-curl -L https://github.com/andreaskoch/gargantua/releases/download/v0.4.1-alpha/gargantua_darwin_amd64 -o gargantua
+curl -L https://github.com/andreaskoch/gargantua/releases/download/v0.5.0-alpha/gargantua_darwin_amd64 -o gargantua
 chmod +x gargantua
 ```
 
 Windows:
 
 ```bash
-curl -L https://github.com/andreaskoch/gargantua/releases/download/v0.4.1-alpha/gargantua_windows_amd64 -o gargantua.exe
+curl -L https://github.com/andreaskoch/gargantua/releases/download/v0.5.0-alpha/gargantua_windows_amd64 -o gargantua.exe
 ```
 
 ## Docker Image

http.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -14,13 +15,18 @@ import (
 )
 
 type Response struct {
+	header      map[string][]string
 	body        []byte
 	statusCode  int
 	startTime   time.Time
 	endTime     time.Time
 	contentType string
 }
 
+func (response *Response) Header() map[string][]string {
+	return response.header
+}
+
 func (response *Response) Body() []byte {
 	return response.body
 }
@@ -59,7 +65,10 @@ func readURL(url url.URL, userAgent string) (Response, error) {
 
 	req.Header.Set("User-Agent", userAgent)
 
-	client := &http.Client{}
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
 	resp, fetchErr := client.Do(req)
 	if fetchErr != nil {
 		return Response{}, fetchErr
@@ -80,7 +89,9 @@ func readURL(url url.URL, userAgent string) (Response, error) {
 	}
 
 	return Response{
-		body:        body,
+		body:   body,
+		header: resp.Header,
+
 		statusCode:  resp.StatusCode,
 		startTime:   startTime,
 		endTime:     endTime,

main.go

@@ -11,7 +11,7 @@ import (
 )
 
 const applicationName = "gargantua"
-const applicationVersion = "v0.4.1-alpha"
+const applicationVersion = "v0.5.0-alpha"
 
 var defaultUserAgent = fmt.Sprintf("%s bot (https://github.com/andreaskoch/gargantua)", applicationName)
 

worker.go

@@ -36,16 +36,19 @@ func executeWork(workerID, numberOfWorkers int, targetURL crawlerUrl, userAgent
 
 	workResult := WorkResult{
 		parentURL: targetURL.parent,
-		url: targetURL.url,
+		url:       targetURL.url,
 
 		workerID:        workerID,
 		numberOfWorkers: numberOfWorkers,
 
 		responseSize: response.Size(),
-		statusCode:   response.StatusCode(),
-		startTime:    response.StartTime(),
-		endTime:      response.EndTime(),
-		contentType:  response.ContentType(),
+		body:         response.Body(),
+		header:       response.Header(),
+
+		statusCode:  response.StatusCode(),
+		startTime:   response.StartTime(),
+		endTime:     response.EndTime(),
+		contentType: response.ContentType(),
 	}
 
 	return workResult

workresult.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"net/url"
+	"strings"
 	"time"
 )
 
@@ -16,20 +17,30 @@ type WorkResult struct {
 	workerID        int
 
 	responseSize int
-	statusCode   int
-	startTime    time.Time
-	endTime      time.Time
-	contentType  string
+	body         []byte
+	header       map[string][]string
+
+	statusCode  int
+	startTime   time.Time
+	endTime     time.Time
+	contentType string
 }
 
 func (workResult WorkResult) String() string {
-	return fmt.Sprintf("#%03d: %03d %9s %15s %20s %20s",
+
+	headers := []string{}
+	for name, value := range workResult.header {
+		headers = append(headers, fmt.Sprintf("'%s:%s'", name, value))
+	}
+
+	return fmt.Sprintf("#%03d: %03d %9s %15s %20s %20s %s",
 		workResult.workerID,
 		workResult.statusCode,
 		fmt.Sprintf("%d", workResult.responseSize),
 		fmt.Sprintf("%fms", workResult.ResponseTime().Seconds()*1000),
 		workResult.url.String(),
 		workResult.parentURL.String(),
+		strings.Join(headers, "|"),
 	)
 }
 
@@ -76,3 +87,11 @@ func (workResult WorkResult) WorkerID() int {
 func (workResult WorkResult) NumberOfWorkers() int {
 	return workResult.numberOfWorkers
 }
+
+func (workResult WorkResult) Body() []byte {
+	return workResult.body
+}
+
+func (workResult WorkResult) Header() map[string][]string {
+	return workResult.header
+}