Skip to content

Update automation script to support copying from local repo. #174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Update automation script to support copying from local repo. #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4b57946
Update automation script to support copying from local repo.
gonzojive Oct 9, 2024
555a72b
add omitempty to local override to avoid clutter in JSON file
gonzojive Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 65 additions & 26 deletions _automation/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"text/tabwriter"
Expand All @@ -36,6 +37,10 @@ type Grammar struct {
Language string `json:"language"`
URL string `json:"url"`
Files []string `json:"files"`
// If specified, files are not downloaded from the remote URL. Instead,
// files are copied from within the local checkout of the git repo
// for the language.
LocalPathOverride string `json:"localPathOverride,omitempty"`
*GrammarVersion
}

Expand Down Expand Up @@ -188,7 +193,9 @@ func (s *UpdateService) Update(ctx context.Context, language string, force bool)

v := grammar.FetchNewVersion()
if v == nil {
if !force {
if grammar.LocalPathOverride != "" {
logger.Warn(fmt.Sprintf("updating grammer from local repository %s", grammar.LocalPathOverride))
} else if !force {
logAndExit(logger, "grammar is not outdated")
} else {
logger.Warn("re-downloading up-to-date grammar")
Expand Down Expand Up @@ -252,19 +259,17 @@ func (s *UpdateService) makeDir(ctx context.Context, path string) {
}

func (s *UpdateService) defaultGrammarDownload(ctx context.Context, g *Grammar) {
url := g.ContentURL()

s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/src/tree_sitter/parser.h", url, g.Revision),
fetcherForFile(g, "src/tree_sitter/parser.h"),
fmt.Sprintf("%s/parser.h", g.Language),
nil,
)

for _, f := range g.Files {
s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/src/%s", url, g.Revision, f),
fetcherForFile(g, "src/"+f),
fmt.Sprintf("%s/%s", g.Language, f),
map[string]string{
`<tree_sitter/parser.h>`: `"parser.h"`,
Expand All @@ -278,14 +283,56 @@ func (s *UpdateService) defaultGrammarDownload(ctx context.Context, g *Grammar)
}
}

func (s *UpdateService) downloadFile(ctx context.Context, url, toPath string, replaceMap map[string]string) {
b := s.fetchFile(ctx, url)
func fetcherForFile(g *Grammar, repoRootRelativePath string) func(ctx context.Context) ([]byte, error) {
if g.LocalPathOverride != "" {
localRelativePath, err := filepath.Localize(repoRootRelativePath)
if err != nil {
return func(_ context.Context) ([]byte, error) {
return nil, fmt.Errorf("failed to localize path %q for local directory %q: %w", repoRootRelativePath, g.LocalPathOverride, err)
}
}
path := filepath.Join(g.LocalPathOverride, localRelativePath)
return func(_ context.Context) ([]byte, error) {
return os.ReadFile(path)
}
}

url := fmt.Sprintf("%s/%s/%s", g.ContentURL(), g.Revision, repoRootRelativePath)

return func(ctx context.Context) ([]byte, error) {
logger := getLogger(ctx).With("url", url)
logger.Debug("fetching")

resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("error downloading file from %s: %w", url, err)
}
defer resp.Body.Close()

if resp.StatusCode != 200 {
return nil, fmt.Errorf("error downloading file from %s: non-200 status code %d", url, resp.StatusCode)
}

b, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("error downloading file from %s: %w", url, err)
}
return b, nil
}

}

func (s *UpdateService) downloadFile(ctx context.Context, fetcher func(context.Context) ([]byte, error), toPath string, replaceMap map[string]string) {
b, err := fetcher(ctx)
if err != nil {
logAndExit(getLogger(ctx), err.Error())
}

for old, new := range replaceMap {
b = bytes.ReplaceAll(b, []byte(old), []byte(new))
}

err := os.WriteFile(toPath, b, 0644)
err = os.WriteFile(toPath, b, 0644)
if err != nil {
logAndExit(getLogger(ctx), err.Error(), "path", toPath)
}
Expand Down Expand Up @@ -330,14 +377,12 @@ func (s *UpdateService) downloadPhp(ctx context.Context, g *Grammar) {
"scanner.h": "common/scanner.h",
}

url := g.ContentURL()

treeSitterFiles := []string{"parser.h", "array.h", "alloc.h"}

for _, f := range treeSitterFiles {
s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/php/src/tree_sitter/%s", url, g.Revision, f),
fetcherForFile(g, "php/src/tree_sitter/"+f),
fmt.Sprintf("%s/tree_sitter/%s", g.Language, f),
nil,
)
Expand All @@ -351,7 +396,7 @@ func (s *UpdateService) downloadPhp(ctx context.Context, g *Grammar) {

s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/%s", url, g.Revision, fp),
fetcherForFile(g, fp),
fmt.Sprintf("%s/%s", g.Language, f),
map[string]string{
`<tree_sitter/parser.h>`: `"parser.h"`,
Expand All @@ -368,7 +413,6 @@ func (s *UpdateService) downloadDockerfile(ctx context.Context, g *Grammar) {
"scanner.c": "src/scanner.c",
}

url := g.ContentURL()
for _, f := range g.Files {
fp, ok := fileMapping[f]
if !ok {
Expand All @@ -377,7 +421,7 @@ func (s *UpdateService) downloadDockerfile(ctx context.Context, g *Grammar) {

s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/%s", url, g.Revision, fp),
fetcherForFile(g, fp),
fmt.Sprintf("%s/%s", g.Language, f),
map[string]string{
`"tree_sitter/parser.h"`: `"parser.h"`,
Expand All @@ -397,7 +441,6 @@ func (s *UpdateService) downloadOcaml(ctx context.Context, g *Grammar) {
"parser.h": "include/tree_sitter/parser.h",
}

url := g.ContentURL()
for _, f := range g.Files {
fp, ok := fileMapping[f]
if !ok {
Expand All @@ -406,7 +449,7 @@ func (s *UpdateService) downloadOcaml(ctx context.Context, g *Grammar) {

s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/%s", url, g.Revision, fp),
fetcherForFile(g, fp),
fmt.Sprintf("%s/%s", g.Language, f),
map[string]string{
`"tree_sitter/alloc.h"`: `"alloc.h"`,
Expand All @@ -420,15 +463,14 @@ func (s *UpdateService) downloadOcaml(ctx context.Context, g *Grammar) {

// typescript is special as it contains 2 different grammars
func (s *UpdateService) downloadTypescript(ctx context.Context, g *Grammar) {
url := g.ContentURL()

langs := []string{"typescript", "tsx"}
for _, lang := range langs {
s.makeDir(ctx, fmt.Sprintf("%s/%s", g.Language, lang))

s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/common/scanner.h", url, g.Revision),
fetcherForFile(g, "common/scanner.h"),
fmt.Sprintf("%s/%s/scanner.h", g.Language, lang),
map[string]string{
`"tree_sitter/parser.h"`: `"parser.h"`,
Expand All @@ -437,15 +479,15 @@ func (s *UpdateService) downloadTypescript(ctx context.Context, g *Grammar) {
)
s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/%s/src/tree_sitter/parser.h", url, g.Revision, lang),
fetcherForFile(g, "src/tree_sitter/parser.h"),
fmt.Sprintf("%s/%s/parser.h", g.Language, lang),
nil,
)

for _, f := range g.Files {
s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/%s/src/%s", url, g.Revision, lang, f),
fetcherForFile(g, fmt.Sprintf("%s/src/%s", lang, f)),
fmt.Sprintf("%s/%s/%s", g.Language, lang, f),
map[string]string{
`"tree_sitter/parser.h"`: `"parser.h"`,
Expand All @@ -459,23 +501,21 @@ func (s *UpdateService) downloadTypescript(ctx context.Context, g *Grammar) {

// markdown is special as it contains 2 different grammars
func (s *UpdateService) downloadMarkdown(ctx context.Context, g *Grammar) {
url := g.ContentURL()

langs := []string{"tree-sitter-markdown", "tree-sitter-markdown-inline"}
for _, lang := range langs {
s.makeDir(ctx, fmt.Sprintf("%s/%s", g.Language, lang))

s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/%s/src/tree_sitter/parser.h", url, g.Revision, lang),
fetcherForFile(g, fmt.Sprintf("%s/src/tree_sitter/parser.h", lang)),
fmt.Sprintf("%s/%s/parser.h", g.Language, lang),
nil,
)

for _, f := range g.Files {
s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/%s/src/%s", url, g.Revision, lang, f),
fetcherForFile(g, fmt.Sprintf("%s/src/%s", lang, f)),
fmt.Sprintf("%s/%s/%s", g.Language, lang, f),
map[string]string{
`"tree_sitter/parser.h"`: `"parser.h"`,
Expand Down Expand Up @@ -519,7 +559,6 @@ func (s *UpdateService) downloadSql(ctx context.Context, g *Grammar) {

s.makeDir(ctx, fmt.Sprintf("%s/tree_sitter", g.Language))

url := g.ContentURL()
for _, f := range g.Files {
fp, ok := fileMapping[f]
if !ok {
Expand All @@ -528,7 +567,7 @@ func (s *UpdateService) downloadSql(ctx context.Context, g *Grammar) {

s.downloadFile(
ctx,
fmt.Sprintf("%s/%s/src/%s", url, g.Revision, fp),
fetcherForFile(g, "src/"+fp),
fmt.Sprintf("%s/%s", g.Language, fp),
nil,
)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/smacker/go-tree-sitter

go 1.20
go 1.23.2

require github.com/stretchr/testify v1.9.0

Expand Down