From 4b57946ec6df9508996420e9fb1bed4b49be03e1 Mon Sep 17 00:00:00 2001 From: Red Daly Date: Wed, 9 Oct 2024 10:47:23 -0700 Subject: [PATCH 1/2] Update automation script to support copying from local repo. --- _automation/main.go | 87 ++++++++++++++++++++++++++++++++------------- go.mod | 2 +- 2 files changed, 63 insertions(+), 26 deletions(-) diff --git a/_automation/main.go b/_automation/main.go index 4ecaa9b..bd873e7 100644 --- a/_automation/main.go +++ b/_automation/main.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "os/exec" + "path/filepath" "strings" "sync" "text/tabwriter" @@ -36,6 +37,10 @@ type Grammar struct { Language string `json:"language"` URL string `json:"url"` Files []string `json:"files"` + // If specified, files are not downloaded from the remote URL. Instead, + // files are copied from within the local checkout of the git repo + // for the language. + LocalPathOverride string `json:"localPathOverride"` *GrammarVersion } @@ -252,11 +257,9 @@ func (s *UpdateService) makeDir(ctx context.Context, path string) { } func (s *UpdateService) defaultGrammarDownload(ctx context.Context, g *Grammar) { - url := g.ContentURL() - s.downloadFile( ctx, - fmt.Sprintf("%s/%s/src/tree_sitter/parser.h", url, g.Revision), + fetcherForFile(g, "src/tree_sitter/parser.h"), fmt.Sprintf("%s/parser.h", g.Language), nil, ) @@ -264,7 +267,7 @@ func (s *UpdateService) defaultGrammarDownload(ctx context.Context, g *Grammar) for _, f := range g.Files { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/src/%s", url, g.Revision, f), + fetcherForFile(g, "src/"+f), fmt.Sprintf("%s/%s", g.Language, f), map[string]string{ ``: `"parser.h"`, @@ -278,14 +281,56 @@ func (s *UpdateService) defaultGrammarDownload(ctx context.Context, g *Grammar) } } -func (s *UpdateService) downloadFile(ctx context.Context, url, toPath string, replaceMap map[string]string) { - b := s.fetchFile(ctx, url) +func fetcherForFile(g *Grammar, repoRootRelativePath string) func(ctx context.Context) ([]byte, error) { + if g.LocalPathOverride != "" { + localRelativePath, err := filepath.Localize(repoRootRelativePath) + if err != nil { + return func(_ context.Context) ([]byte, error) { + return nil, fmt.Errorf("failed to localize path %q for local directory %q: %w", repoRootRelativePath, g.LocalPathOverride, err) + } + } + path := filepath.Join(g.LocalPathOverride, localRelativePath) + return func(_ context.Context) ([]byte, error) { + return os.ReadFile(path) + } + } + + url := fmt.Sprintf("%s/%s/%s", g.ContentURL(), g.Revision, repoRootRelativePath) + + return func(ctx context.Context) ([]byte, error) { + logger := getLogger(ctx).With("url", url) + logger.Debug("fetching") + + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("error downloading file from %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("error downloading file from %s: non-200 status code %d", url, resp.StatusCode) + } + + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error downloading file from %s: %w", url, err) + } + return b, nil + } + +} + +func (s *UpdateService) downloadFile(ctx context.Context, fetcher func(context.Context) ([]byte, error), toPath string, replaceMap map[string]string) { + b, err := fetcher(ctx) + if err != nil { + logAndExit(getLogger(ctx), err.Error()) + } for old, new := range replaceMap { b = bytes.ReplaceAll(b, []byte(old), []byte(new)) } - err := os.WriteFile(toPath, b, 0644) + err = os.WriteFile(toPath, b, 0644) if err != nil { logAndExit(getLogger(ctx), err.Error(), "path", toPath) } @@ -330,14 +375,12 @@ func (s *UpdateService) downloadPhp(ctx context.Context, g *Grammar) { "scanner.h": "common/scanner.h", } - url := g.ContentURL() - treeSitterFiles := []string{"parser.h", "array.h", "alloc.h"} for _, f := range treeSitterFiles { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/php/src/tree_sitter/%s", url, g.Revision, f), + fetcherForFile(g, "php/src/tree_sitter/"+f), fmt.Sprintf("%s/tree_sitter/%s", g.Language, f), nil, ) @@ -351,7 +394,7 @@ func (s *UpdateService) downloadPhp(ctx context.Context, g *Grammar) { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/%s", url, g.Revision, fp), + fetcherForFile(g, fp), fmt.Sprintf("%s/%s", g.Language, f), map[string]string{ ``: `"parser.h"`, @@ -368,7 +411,6 @@ func (s *UpdateService) downloadDockerfile(ctx context.Context, g *Grammar) { "scanner.c": "src/scanner.c", } - url := g.ContentURL() for _, f := range g.Files { fp, ok := fileMapping[f] if !ok { @@ -377,7 +419,7 @@ func (s *UpdateService) downloadDockerfile(ctx context.Context, g *Grammar) { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/%s", url, g.Revision, fp), + fetcherForFile(g, fp), fmt.Sprintf("%s/%s", g.Language, f), map[string]string{ `"tree_sitter/parser.h"`: `"parser.h"`, @@ -397,7 +439,6 @@ func (s *UpdateService) downloadOcaml(ctx context.Context, g *Grammar) { "parser.h": "include/tree_sitter/parser.h", } - url := g.ContentURL() for _, f := range g.Files { fp, ok := fileMapping[f] if !ok { @@ -406,7 +447,7 @@ func (s *UpdateService) downloadOcaml(ctx context.Context, g *Grammar) { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/%s", url, g.Revision, fp), + fetcherForFile(g, fp), fmt.Sprintf("%s/%s", g.Language, f), map[string]string{ `"tree_sitter/alloc.h"`: `"alloc.h"`, @@ -420,7 +461,6 @@ func (s *UpdateService) downloadOcaml(ctx context.Context, g *Grammar) { // typescript is special as it contains 2 different grammars func (s *UpdateService) downloadTypescript(ctx context.Context, g *Grammar) { - url := g.ContentURL() langs := []string{"typescript", "tsx"} for _, lang := range langs { @@ -428,7 +468,7 @@ func (s *UpdateService) downloadTypescript(ctx context.Context, g *Grammar) { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/common/scanner.h", url, g.Revision), + fetcherForFile(g, "common/scanner.h"), fmt.Sprintf("%s/%s/scanner.h", g.Language, lang), map[string]string{ `"tree_sitter/parser.h"`: `"parser.h"`, @@ -437,7 +477,7 @@ func (s *UpdateService) downloadTypescript(ctx context.Context, g *Grammar) { ) s.downloadFile( ctx, - fmt.Sprintf("%s/%s/%s/src/tree_sitter/parser.h", url, g.Revision, lang), + fetcherForFile(g, "src/tree_sitter/parser.h"), fmt.Sprintf("%s/%s/parser.h", g.Language, lang), nil, ) @@ -445,7 +485,7 @@ func (s *UpdateService) downloadTypescript(ctx context.Context, g *Grammar) { for _, f := range g.Files { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/%s/src/%s", url, g.Revision, lang, f), + fetcherForFile(g, fmt.Sprintf("%s/src/%s", lang, f)), fmt.Sprintf("%s/%s/%s", g.Language, lang, f), map[string]string{ `"tree_sitter/parser.h"`: `"parser.h"`, @@ -459,15 +499,13 @@ func (s *UpdateService) downloadTypescript(ctx context.Context, g *Grammar) { // markdown is special as it contains 2 different grammars func (s *UpdateService) downloadMarkdown(ctx context.Context, g *Grammar) { - url := g.ContentURL() - langs := []string{"tree-sitter-markdown", "tree-sitter-markdown-inline"} for _, lang := range langs { s.makeDir(ctx, fmt.Sprintf("%s/%s", g.Language, lang)) s.downloadFile( ctx, - fmt.Sprintf("%s/%s/%s/src/tree_sitter/parser.h", url, g.Revision, lang), + fetcherForFile(g, fmt.Sprintf("%s/src/tree_sitter/parser.h", lang)), fmt.Sprintf("%s/%s/parser.h", g.Language, lang), nil, ) @@ -475,7 +513,7 @@ func (s *UpdateService) downloadMarkdown(ctx context.Context, g *Grammar) { for _, f := range g.Files { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/%s/src/%s", url, g.Revision, lang, f), + fetcherForFile(g, fmt.Sprintf("%s/src/%s", lang, f)), fmt.Sprintf("%s/%s/%s", g.Language, lang, f), map[string]string{ `"tree_sitter/parser.h"`: `"parser.h"`, @@ -519,7 +557,6 @@ func (s *UpdateService) downloadSql(ctx context.Context, g *Grammar) { s.makeDir(ctx, fmt.Sprintf("%s/tree_sitter", g.Language)) - url := g.ContentURL() for _, f := range g.Files { fp, ok := fileMapping[f] if !ok { @@ -528,7 +565,7 @@ func (s *UpdateService) downloadSql(ctx context.Context, g *Grammar) { s.downloadFile( ctx, - fmt.Sprintf("%s/%s/src/%s", url, g.Revision, fp), + fetcherForFile(g, "src/"+fp), fmt.Sprintf("%s/%s", g.Language, fp), nil, ) diff --git a/go.mod b/go.mod index baed079..d7755b7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/smacker/go-tree-sitter -go 1.20 +go 1.23.2 require github.com/stretchr/testify v1.9.0 From 555a72b9fd5f0ec0022b438bec3d76431bc9896c Mon Sep 17 00:00:00 2001 From: Red Daly Date: Wed, 9 Oct 2024 10:53:27 -0700 Subject: [PATCH 2/2] add omitempty to local override to avoid clutter in JSON file --- _automation/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/_automation/main.go b/_automation/main.go index bd873e7..53f8a1b 100644 --- a/_automation/main.go +++ b/_automation/main.go @@ -40,7 +40,7 @@ type Grammar struct { // If specified, files are not downloaded from the remote URL. Instead, // files are copied from within the local checkout of the git repo // for the language. - LocalPathOverride string `json:"localPathOverride"` + LocalPathOverride string `json:"localPathOverride,omitempty"` *GrammarVersion } @@ -193,7 +193,9 @@ func (s *UpdateService) Update(ctx context.Context, language string, force bool) v := grammar.FetchNewVersion() if v == nil { - if !force { + if grammar.LocalPathOverride != "" { + logger.Warn(fmt.Sprintf("updating grammer from local repository %s", grammar.LocalPathOverride)) + } else if !force { logAndExit(logger, "grammar is not outdated") } else { logger.Warn("re-downloading up-to-date grammar")