Skip to content

Commit 66867ba

Browse files
marcotunamarcotuna
committed
feat: add sitemap
1 parent 8a5d347 commit 66867ba

2 files changed

Lines changed: 346 additions & 0 deletions

File tree

‎cmd/wp-go-static/commands/sitemap.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package commands
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"net/url"
7+
"strings"
8+
9+
goSitemap "wp-go-static/pkg/sitemap"
10+
11+
"github.com/spf13/cobra"
12+
"github.com/spf13/viper"
13+
)
14+
15+
type SitemapConfig struct {
16+
Dir string `mapstructure:"dir"`
17+
URL string `mapstructure:"url"`
18+
ReplaceURL string `mapstructure:"replace-url"`
19+
File string `mapstructure:"file"`
20+
}
21+
22+
// SitemapCmd ...
23+
var SitemapCmd = &cobra.Command{
24+
Use: "sitemap",
25+
Short: "Create sitemap from the Wordpress website",
26+
RunE: sitemapCmdF,
27+
}
28+
29+
func init() {
30+
// Define command-line flags
31+
SitemapCmd.PersistentFlags().String("dir", "dump", "directory to save downloaded files")
32+
SitemapCmd.PersistentFlags().String("url", "", "URL to scrape")
33+
SitemapCmd.PersistentFlags().String("replace-url", "", "Replace with a specific url")
34+
SitemapCmd.PersistentFlags().String("file", "sitemap.xml", "Output sitemap file name")
35+
SitemapCmd.MarkFlagRequired("url")
36+
37+
// Bind command-line flags to Viper
38+
err := viper.BindPFlags(SitemapCmd.PersistentFlags())
39+
if err != nil {
40+
log.Fatal(err)
41+
}
42+
43+
RootCmd.AddCommand(SitemapCmd)
44+
}
45+
46+
func sitemapCmdF(command *cobra.Command, args []string) error {
47+
sitemapConfig := SitemapConfig{}
48+
viper.Unmarshal(&sitemapConfig)
49+
50+
smap, err := goSitemap.Get(sitemapConfig.URL, nil)
51+
if err != nil {
52+
fmt.Println(err)
53+
}
54+
55+
for i := range smap.URL {
56+
// Replace the URL with the url from the replace-url argument
57+
// Only with the URL part, persist the URL path and query
58+
if sitemapConfig.ReplaceURL != "" {
59+
currentURL, _ := url.Parse(sitemapConfig.URL)
60+
61+
optionList := []string{
62+
fmt.Sprintf(`http://%s`, currentURL.Host),
63+
fmt.Sprintf(`http:\/\/%s`, currentURL.Host),
64+
fmt.Sprintf(`https://%s`, currentURL.Host),
65+
fmt.Sprintf(`https:\/\/%s`, currentURL.Host),
66+
}
67+
68+
for _, option := range optionList {
69+
if i >= len(smap.URL) {
70+
fmt.Println("Index out of range for smap.URL")
71+
break
72+
}
73+
smap.URL[i].Loc = strings.ReplaceAll(string(smap.URL[i].Loc), option, sitemapConfig.ReplaceURL)
74+
75+
// for j := range smap.Image {
76+
// if i >= len(smap.URL) {
77+
// fmt.Println("Index out of range for smap.URL")
78+
// break
79+
// }
80+
// if j >= len(smap.URL[i].Image) {
81+
// fmt.Println("Index out of range for smap.URL[i].Image")
82+
// break
83+
// }
84+
// smap.URL[i].Image[j].Loc = strings.ReplaceAll(string(smap.URL[i].Image[j].Loc), option, sitemapConfig.ReplaceURL)
85+
// }
86+
}
87+
}
88+
}
89+
90+
// Print the Sitemap
91+
printSmap, err := smap.Print()
92+
if err != nil {
93+
return err
94+
}
95+
96+
fmt.Printf("%s\n", printSmap)
97+
98+
// Write the Sitemap to a file
99+
if sitemapConfig.File != "" {
100+
fmt.Printf("Writing sitemap to %s/%s\n", sitemapConfig.Dir, sitemapConfig.File)
101+
return smap.Save(sitemapConfig.Dir, sitemapConfig.File)
102+
}
103+
104+
return nil
105+
}

‎pkg/sitemap/sitemap.go‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
package sitemap
2+
3+
import (
4+
"encoding/xml"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
"os"
9+
"time"
10+
)
11+
12+
// Index is a structure of <sitemapindex>
13+
type Index struct {
14+
XMLName xml.Name `xml:"sitemapindex"`
15+
Sitemap []parts `xml:"sitemap"`
16+
}
17+
18+
// parts is a structure of <sitemap> in <sitemapindex>
19+
type parts struct {
20+
Loc string `xml:"loc"`
21+
LastMod string `xml:"lastmod"`
22+
}
23+
24+
// Sitemap is a structure of <sitemap>
25+
type Sitemap struct {
26+
// Xsi string `xml:"xsi,attr"`
27+
// Image string `xml:"image,attr"`
28+
// SchemaLocation string `xml:"schemaLocation,attr"`
29+
// Xmlns string `xml:"xmlns,attr"`
30+
XMLName xml.Name `xml:"urlset"`
31+
URL []URL `xml:"url"`
32+
}
33+
34+
// URL is a structure of <url> in <sitemap>
35+
type URL struct {
36+
Loc string `xml:"loc"`
37+
LastMod string `xml:"lastmod,omitempty"`
38+
ChangeFreq string `xml:"changefreq,omitempty"`
39+
Priority float32 `xml:"priority,omitempty"`
40+
// Image []Image `xml:"image,omitempty"`
41+
}
42+
43+
// Image is a structure of <image> in <url>
44+
type Image struct {
45+
Loc string `xml:"loc,omitempty"`
46+
Title string `xml:"title,omitempty"`
47+
Caption string `xml:"caption,omitempty"`
48+
GeoLoc string `xml:"geolocation,omitempty"`
49+
License string `xml:"license,omitempty"`
50+
}
51+
52+
var (
53+
// fetch is page acquisition function
54+
fetch = func(URL string, options interface{}) ([]byte, error) {
55+
var body []byte
56+
57+
res, err := http.Get(URL)
58+
if err != nil {
59+
return body, err
60+
}
61+
defer res.Body.Close()
62+
63+
return io.ReadAll(res.Body)
64+
}
65+
66+
// Time interval to be used in Index.get
67+
interval = time.Second
68+
)
69+
70+
/*
71+
Get is fetch and parse sitemap.xml/sitemapindex.xml
72+
73+
If sitemap.xml or sitemapindex.xml has some problems, This function return error.
74+
75+
・When sitemap.xml/sitemapindex.xml could not retrieved.
76+
・When sitemap.xml/sitemapindex.xml is empty.
77+
・When sitemap.xml/sitemapindex.xml has format problems.
78+
・When sitemapindex.xml contains a sitemap.xml URL that cannot be retrieved.
79+
・When sitemapindex.xml contains a sitemap.xml that is empty
80+
・When sitemapindex.xml contains a sitemap.xml that has format problems.
81+
82+
If you want to ignore these errors, use the ForceGet function.
83+
*/
84+
func Get(URL string, options interface{}) (Sitemap, error) {
85+
data, err := fetch(URL, options)
86+
if err != nil {
87+
return Sitemap{}, err
88+
}
89+
90+
idx, idxErr := ParseIndex(data)
91+
smap, smapErr := Parse(data)
92+
93+
if idxErr != nil && smapErr != nil {
94+
if idxErr != nil {
95+
err = idxErr
96+
} else {
97+
err = smapErr
98+
}
99+
return Sitemap{}, fmt.Errorf("URL is not a sitemap or sitemapindex: %v", err)
100+
} else if idxErr != nil {
101+
return smap, nil
102+
}
103+
104+
smap, err = idx.get(options, false)
105+
if err != nil {
106+
return Sitemap{}, err
107+
}
108+
109+
return smap, nil
110+
}
111+
112+
/*
113+
ForceGet is fetch and parse sitemap.xml/sitemapindex.xml.
114+
The difference with the Get function is that it ignores some errors.
115+
116+
Errors to Ignore:
117+
118+
・When sitemapindex.xml contains a sitemap.xml URL that cannot be retrieved.
119+
・When sitemapindex.xml contains a sitemap.xml that is empty
120+
・When sitemapindex.xml contains a sitemap.xml that has format problems.
121+
122+
Errors not to Ignore:
123+
124+
・When sitemap.xml/sitemapindex.xml could not retrieved.
125+
・When sitemap.xml/sitemapindex.xml is empty.
126+
・When sitemap.xml/sitemapindex.xml has format problems.
127+
128+
If you want **not** to ignore some errors, use the Get function.
129+
*/
130+
func ForceGet(URL string, options interface{}) (Sitemap, error) {
131+
data, err := fetch(URL, options)
132+
if err != nil {
133+
return Sitemap{}, err
134+
}
135+
136+
idx, idxErr := ParseIndex(data)
137+
smap, smapErr := Parse(data)
138+
139+
if idxErr != nil && smapErr != nil {
140+
if idxErr != nil {
141+
err = idxErr
142+
} else {
143+
err = smapErr
144+
}
145+
return Sitemap{}, fmt.Errorf("URL is not a sitemap or sitemapindex: %v", err)
146+
} else if idxErr != nil {
147+
return smap, nil
148+
}
149+
150+
smap, err = idx.get(options, true)
151+
if err != nil {
152+
return Sitemap{}, err
153+
}
154+
155+
return smap, nil
156+
}
157+
158+
// Get Sitemap data from sitemapindex file
159+
func (idx *Index) get(options interface{}, ignoreErr bool) (Sitemap, error) {
160+
var smap Sitemap
161+
162+
for _, s := range idx.Sitemap {
163+
time.Sleep(interval)
164+
data, err := fetch(s.Loc, options)
165+
if !ignoreErr && err != nil {
166+
return smap, fmt.Errorf("failed to retrieve %s in sitemapindex.xml: %v", s.Loc, err)
167+
}
168+
169+
err = xml.Unmarshal(data, &smap)
170+
if !ignoreErr && err != nil {
171+
return smap, fmt.Errorf("failed to parse %s in sitemapindex.xml: %v", s.Loc, err)
172+
}
173+
}
174+
175+
return smap, nil
176+
}
177+
178+
// Parse create Sitemap data from text
179+
func Parse(data []byte) (Sitemap, error) {
180+
var smap Sitemap
181+
if len(data) == 0 {
182+
return smap, fmt.Errorf("sitemap.xml is empty")
183+
}
184+
185+
err := xml.Unmarshal(data, &smap)
186+
return smap, err
187+
}
188+
189+
// ParseIndex create Index data from text
190+
func ParseIndex(data []byte) (Index, error) {
191+
var idx Index
192+
if len(data) == 0 {
193+
return idx, fmt.Errorf("sitemapindex.xml is empty")
194+
}
195+
196+
err := xml.Unmarshal(data, &idx)
197+
return idx, err
198+
}
199+
200+
// SetInterval change Time interval to be used in Index.get
201+
func SetInterval(time time.Duration) {
202+
interval = time
203+
}
204+
205+
// SetFetch change fetch closure
206+
func SetFetch(f func(URL string, options interface{}) ([]byte, error)) {
207+
fetch = f
208+
}
209+
210+
// Print shows the sitemap from Sitemap struct
211+
func (smap *Sitemap) Print() ([]byte, error) {
212+
return xml.MarshalIndent(smap, "", " ")
213+
}
214+
215+
// Save creates the sitemap from Sitemap struct and save it to file
216+
func (smap *Sitemap) Save(dir, file string) error {
217+
data, err := smap.Print()
218+
if err != nil {
219+
return err
220+
}
221+
222+
// Add the xml header
223+
data = append([]byte(xml.Header), data...)
224+
225+
// Create directory if it does not exist
226+
if _, err := os.Stat(dir); os.IsNotExist(err) {
227+
os.Mkdir(dir, 0755)
228+
}
229+
230+
_, err = os.Create(fmt.Sprintf("%s/%s", dir, file))
231+
if err != nil {
232+
return err
233+
}
234+
235+
err = os.WriteFile(fmt.Sprintf("%s/%s", dir, file), data, 0644)
236+
if err != nil {
237+
return err
238+
}
239+
240+
return nil
241+
}

0 commit comments

Comments
 (0)