完成数据持久化

Author: littleghost2016

README.md

@@ -1,2 +1,55 @@
 # xdncov
-本仓库仅用作练习gocolly爬虫，不可用于其他用途，若出现任何问题，与本人无关。
+本仓库仅用作练习`gocolly`爬虫，不可用于其他用途，若出现任何问题，与本人无关。
+
+## 准备工作
+
+### 文件准备
+
+创建`configs`文件夹，按照`configs`文件夹里面的`example.config`，改成自己的配置文件并更改文件名以`.toml`为后缀。
+
+例如`11111111111.toml`
+
+```toml
+# 已返校同学可只更改前三项
+name     = "张三"        # 姓名
+id       = "11111111111" # 学号
+password = "11111111"    # 密码
+
+# 其他内容可不更改
+province = "陕西省"
+city     = "西安市"
+district = "长安区"
+address  = "陕西省西安市西沣路兴隆段266号"
+ymtys    = 0 # 一码通颜色
+tw       = 1 # 体温
+sfzx     = 1 # 是否在校
+sfcyglq  = 0 # 是否处于隔离期
+sfyzz    = 0 # 是否有症状
+
+# 以下内容可能会由程序自动更改
+cookie   = "" # 用作持久化
+path     = "" # 文件保存路径
+```
+
+## 执行
+
+下载对应平台的可执行文件，执行即可。
+
+### PowerShell
+
+```powershell
+.\xdncov.exe
+```
+
+### bash/zsh
+
+```bash
+./xdncov
+```
+
+## 后续工作
+
+- [x] 持久化存储
+- [ ] 定时执行
+- [ ] Toml添加最后一次提交时间
+- [ ] 邮件提醒

compile.ps1

@@ -0,0 +1,9 @@
+$env:GOOS="windows"
+$env:GOARCH="amd64"
+go build -o xdncov_windows_amd64.exe
+
+$env:GOOS="linux"
+go build -o xdncov_linux_amd64
+
+$env:GOOS="darwin"
+go build -o xdncov_darwin_amd64

config.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"path"
+
+	"github.com/BurntSushi/toml"
+)
+
+// StudentConfig 解析toml配置文件
+type StudentConfig struct {
+	Name     string `toml:"name"`
+	ID       int    `toml:"id"`
+	Password string `toml:"password"`
+	Province string `toml:"province"`
+	City     string `toml:"city"`
+	District string `toml:"district"`
+	Address  string `toml:"address"`
+	Ymtys    int    `toml:"ymtys"`
+	Tw       int    `toml:"tw"`
+	Sfzx     int    `toml:"sfzx"`
+	Sfcyglq  int    `toml:"sfcyglq"`
+	Sfyzz    int    `toml:"sfyzz"`
+	Cookie   string `toml:"cookie"`
+	Path     string `toml:"path"`
+}
+
+// CollectConfigs 收集toml配置文件
+func CollectConfigs(configDirectoryPath string) (studentConfigSlice []StudentConfig) {
+	allFiles, err := ioutil.ReadDir(configDirectoryPath)
+	if err != nil {
+		fmt.Println("ioutil.ReadDir has error.", err)
+	}
+
+	for _, eachFileName := range allFiles {
+
+		filenameWithSuffix := path.Base(eachFileName.Name())
+		//filenameWithSuffix: 文件名带后缀。
+		// fmt.Println("filenameWithSuffix =", filenameWithSuffix)
+
+		fileSuffix := path.Ext(filenameWithSuffix)
+		//fileSuffix: 文件后缀
+		// fmt.Println("fileSuffix =", fileSuffix)
+		if fileSuffix == ".toml" {
+			configPath := fmt.Sprintf("%s/%s", configDirectoryPath, eachFileName.Name())
+			studentConfigSlice = append(studentConfigSlice, ReadConfig(configPath))
+		}
+	}
+
+	return studentConfigSlice
+}
+
+// ReadConfig 读取配置文件
+func ReadConfig(configPath string) (tempConfig StudentConfig) {
+	if _, err := toml.DecodeFile(configPath, &tempConfig); err != nil {
+		log.Fatalln(err)
+	}
+	tempConfig.Path = configPath
+
+	return
+}
+
+// UpdateConfig 更新配置文件
+func UpdateConfig(newConfig StudentConfig) {
+	buf := new(bytes.Buffer)
+	if err := toml.NewEncoder(buf).Encode(newConfig); err != nil {
+		log.Fatal(err)
+	}
+
+	if ioutil.WriteFile(newConfig.Path, buf.Bytes(), 0644) == nil {
+		fmt.Println("写入文件成功:", newConfig.Path)
+	}
+}

configs/example.config

@@ -0,0 +1,19 @@
+# 已返校同学可只更改前三项
+name     = "张三"        # 姓名
+id       = "11111111111" # 学号
+password = "11111111"    # 密码
+
+# 其他内容可不更改
+province = "陕西省"
+city     = "西安市"
+district = "长安区"
+address  = "陕西省西安市西沣路兴隆段266号"
+ymtys    = 0 # 一码通颜色
+tw       = 1 # 体温
+sfzx     = 1 # 是否在校
+sfcyglq  = 0 # 是否处于隔离期
+sfyzz    = 0 # 是否有症状
+
+# 以下内容可能会由程序自动更改
+cookie   = "" # 用作持久化
+path     = "" # 文件保存路径

go.mod

@@ -0,0 +1,17 @@
+module xdncov
+
+go 1.14
+
+require (
+	github.com/BurntSushi/toml v0.3.1
+	github.com/PuerkitoBio/goquery v1.5.1 // indirect
+	github.com/antchfx/htmlquery v1.2.3 // indirect
+	github.com/antchfx/xmlquery v1.2.4 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gocolly/colly v1.2.0
+	github.com/kennygrant/sanitize v1.2.4 // indirect
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/temoto/robotstxt v1.1.1 // indirect
+	golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect
+	google.golang.org/appengine v1.6.6 // indirect
+)

go.sum

@@ -0,0 +1,51 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
+github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
+github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4=
+github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
+github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
+github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
+github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
+github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
+github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=

httpRequest.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+
+	"github.com/gocolly/colly"
+	"github.com/gocolly/colly/storage"
+)
+
+const (
+	// BaseURL 域名URL
+	BaseURL = "https://xxcapp.xidian.edu.cn"
+	// LoginURL 登录URL
+	LoginURL = "https://xxcapp.xidian.edu.cn/uc/wap/login/check"
+	// SaveURL 提交结果URL
+	SaveURL = "https://xxcapp.xidian.edu.cn/ncov/wap/open-report/save"
+	// MyUserAgent 模拟手机UA
+	MyUserAgent = "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.45 Mobile Safari/537.36 Edg/84.0.522.20"
+)
+
+// PostSaveForm 提交晨午检表单
+func PostSaveForm(newClient *colly.Collector, config StudentConfig) {
+	savePostForm := map[string]string{
+		"province": config.Province,
+		"city":     config.City,
+		"district": config.District,
+		"address":  config.Address,
+		"ymtys":    strconv.Itoa(config.Ymtys),
+		"tw":       strconv.Itoa(config.Tw),
+		"sfzx":     strconv.Itoa(config.Sfzx),
+		"sfcyglq":  strconv.Itoa(config.Sfcyglq),
+		"sfyzz":    strconv.Itoa(config.Sfyzz),
+	}
+	err := newClient.Post(SaveURL, savePostForm)
+	if err != nil {
+		log.Fatal(2, err)
+	}
+}
+
+// SignIn HTTP主要步骤
+func SignIn(config StudentConfig) {
+	firstPostClient := colly.NewCollector()
+	firstPostClient.UserAgent = MyUserAgent
+
+	firstPostSuccessFlag := false
+	if config.Cookie != "" {
+		firstPostClient.OnRequest(func(request *colly.Request) {
+			firstPostClient.SetCookies(BaseURL, storage.UnstringifyCookies(config.Cookie))
+		})
+
+	}
+	firstPostClient.OnResponse(func(r *colly.Response) {
+		tempResponse := UnmarshalHTTPResponse(r.Body)
+		if tempResponse.M != "" {
+			if tempResponse.M == "操作成功" || tempResponse.M == "您已上报过" {
+				firstPostSuccessFlag = true
+				fmt.Println(config.ID, "第一次提交即成功")
+			}
+		}
+	})
+	PostSaveForm(firstPostClient, config)
+
+	if !firstPostSuccessFlag {
+		firstLoginClient := firstPostClient.Clone()
+		loginFlag := Login(firstLoginClient, strconv.Itoa(config.ID), config.Password)
+		if loginFlag {
+			fmt.Println(config.ID, "登陆成功")
+		}
+
+		secondPostClient := firstLoginClient.Clone()
+		secondPostClient.OnResponse(func(response *colly.Response) {
+			newCookie := storage.StringifyCookies(secondPostClient.Cookies(response.Request.URL.String()))
+			config.Cookie = newCookie
+			UpdateConfig(config)
+			fmt.Println(string(response.Body))
+		})
+		PostSaveForm(secondPostClient, config)
+	}
+}
+
+// Login 当持久化未能通过时，模拟登录以获得cookie
+func Login(newClient *colly.Collector, id string, password string) (loginFlag bool) {
+	loginFlag = false
+
+	newClient.OnResponse(func(r *colly.Response) {
+		tempResponse := UnmarshalHTTPResponse(r.Body)
+		if tempResponse.M != "" {
+			if tempResponse.M == "操作成功" {
+				loginFlag = true
+			}
+		}
+	})
+
+	err := newClient.Post(LoginURL, map[string]string{
+		"username": id,
+		"password": password,
+	})
+	if err != nil {
+		log.Fatal(1, err)
+	}
+
+	return
+}

util.go

@@ -0,0 +1,22 @@
+package main
+
+import "encoding/json"
+
+// HTTPResponse 解析返回的json格式的消息
+type HTTPResponse struct {
+	E int         `json:"e"`
+	M string      `json:"m"`
+	D EmptyStruct `json:"d"`
+}
+
+// EmptyStruct HTTPResponse.D
+type EmptyStruct struct {
+}
+
+// unmarshalHTTPResponse 解析json消息
+func UnmarshalHTTPResponse(response []byte) (newHTTPResponse HTTPResponse) {
+	newHTTPResponse = HTTPResponse{}
+	json.Unmarshal(response, &newHTTPResponse)
+
+	return
+}

xdncov.go

@@ -0,0 +1,12 @@
+package main
+
+func main() {
+
+	studentConfigSlice := CollectConfigs("./configs")
+
+	for _, eachConfig := range studentConfigSlice {
+		// go signIn(eachConfig)
+		SignIn(eachConfig)
+	}
+
+}