ocaml-tree-sitter-gen-ocaml

#! /usr/bin/env bash
#
# Generate OCaml parsers
#
# For lang=ruby, the file hierarchy we create looks like this:
#
# ocaml-src
# ├── bin
# │   ├── dune
# │   └── Main.ml
# ├── lib
# │   ├── bindings.c
# │   ├── Boilerplate.ml
# │   ├── CST.ml
# │   ├── dune
# │   ├── Parse.ml
# │   ├── Parse.mli
# │   ├── parser.c
# │   ├── scanner.cc
# │   └── tree_sitter
# │       └── parser.h
# └── tree-sitter-ruby.opam
#
set -eu -o pipefail

project_root=$(git rev-parse --show-toplevel)

# The ocaml-tree-sitter executable and the runtime library are assumed to
# to have been installed with 'make install'
#
ocaml_tree_sitter=$(which ocaml-tree-sitter)

default_dst_dir="ocaml-src"
default_src_dir="src"
default_lang="lang"
default_package="tree-sitter-lang"

usage() {
  cat <<EOF
Usage: $(basename "$0") [OPTIONS]

Call ocaml-tree-sitter to derive an OCaml parsing library and executable
from a tree-sitter grammar in json format, normally generated by
tree-sitter as 'src/grammar.json'.

Options:
  --dst DST_DIR
      Specify the output directory. Default: $default_dst_dir
  --help
      Show this help message and exit.
  --lang LANG
      Name of the programming language. It will be part of the name
      of the library SUPERNAME.LANG and of the OCaml module.
      Case conversion and conversions between dashes and underscores will
      take place as needed.
      Default: $default_lang
  --src SRC_DIR
      Location of the 'src' folder with some of its contents generated
      by tree-sitter. It must contain 'grammar.json', 'parser.c',
      and optionally other C files ('scanner.c' or 'scanner.cc') needed
      to build the C parser. Default: $src_dir
  --package PKG
      The Dune package name to which the library will belong.
      The library name will be PKG.LANG where LANG is the language name.
      Default: $default_package.
  --trace
      Print debugging info during parsing.
  --tree-sitter-libdir LIBDIR
      Where the tree-sitter runtime library was installed.
      Default:
EOF
}

error() {
  cat >&2 <<EOF
Error: $*
EOF
  exit 1
}

test -x "$ocaml_tree_sitter" || error "missing executable $ocaml_tree_sitter"

lang="$default_lang"
package="$default_package"
src_dir="$default_src_dir"
dst_dir="$default_dst_dir"
trace_option=()

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dst)
      dst_dir="$2"
      shift
      ;;
    --help)
      usage
      exit 0
      ;;
    --lang)
      lang="$2"
      shift
      ;;
    --package)
      package="$2"
      shift
      ;;
    --src)
      src_dir="$2"
      shift
      ;;
    --trace)
      trace_option="--trace"
      ;;
    *)
      error "Unsupported argument: $1"
  esac
  shift
done

lang_dashes=$(echo "$lang" | tr 'A-Z_' 'a-z-')
lang_underscores=$(echo "$lang" | tr 'A-Z-' 'a-z_')

# Copy what we need to the ocaml-src folder. We need at least to copy
# the C source and headers.
#
rm -rf "$dst_dir"
mkdir -p "$dst_dir"/lib

# Build the lists of C and C++ files to compile, without their extension.
#
c_files="parser bindings"
cxx_files=""

cp "$src_dir"/parser.c "$dst_dir"/lib

if [[ -e "$src_dir"/scanner.c ]]; then
  cp "$src_dir"/scanner.c "$dst_dir"/lib
  c_files="scanner $c_files"
fi

if [[ -e "$src_dir"/scanner.cc ]]; then
  cp "$src_dir"/scanner.cc "$dst_dir"/lib
  cxx_files="scanner"
fi

# Copy C header files
#
shopt -s nullglob
for x in "$src_dir"/*.h; do cp "$x" "$dst_dir"/lib; done
cp -a "$src_dir"/tree_sitter "$dst_dir"/lib/tree_sitter

# Generate the OCaml code needed to parse the examples/*.out json files.
#
"$ocaml_tree_sitter" \
  gen \
  "$lang_underscores" \
  "$src_dir"/grammar.json \
  -d "$dst_dir" \
  "${trace_option[@]}"

cat > "$dst_dir"/lib/bindings.c <<EOF
/*
  Generated by ocaml-tree-sitter for $lang_underscores.
*/

#include <string.h>
#include <tree_sitter/api.h>

#include <caml/alloc.h>
#include <caml/bigarray.h>
#include <caml/callback.h>
#include <caml/custom.h>
#include <caml/memory.h>
#include <caml/mlvalues.h>
#include <caml/threads.h>

// Implemented by parser.c
TSLanguage *tree_sitter_${lang_underscores}();

typedef struct _parser {
  TSParser *parser;
} parser_W;

static void finalize_parser(value v) {
  parser_W *p;
  p = (parser_W *)Data_custom_val(v);
  ts_parser_delete(p->parser);
}

static struct custom_operations parser_custom_ops = {
  .identifier = "parser handling",
  .finalize = finalize_parser,
  .compare = custom_compare_default,
  .hash = custom_hash_default,
  .serialize = custom_serialize_default,
  .deserialize = custom_deserialize_default
};

// OCaml function
CAMLprim value octs_create_parser_${lang_underscores}(value unit) {
  CAMLparam0();
  CAMLlocal1(v);

  parser_W parserWrapper;
  TSParser *parser = ts_parser_new();
  parserWrapper.parser = parser;

  v = caml_alloc_custom(&parser_custom_ops, sizeof(parser_W), 0, 1);
  memcpy(Data_custom_val(v), &parserWrapper, sizeof(parser_W));
  ts_parser_set_language(parser, tree_sitter_${lang_underscores}());
  CAMLreturn(v);
};
EOF

cat > "$dst_dir"/lib/dune <<EOF
; required to install tree_sitter/parser.h
(include_subdirs qualified)

(library
  (public_name $package.${lang_dashes})
  (name tree_sitter_${lang_underscores})
  (libraries atdgen-runtime tree-sitter.run)

  ; A copy of the C headers for the tree-sitter library is found locally.
  ; This is because it's important to use the right version of 'parser.h'.
  ;
  (foreign_stubs
    (language c)
    (names ${c_files})
    (flags -std=c99
           -fPIC
           -I %{env:TREESITTER_INCDIR=/usr/local/include}
           -I .)
  )

  ; TREESITTER_LIBDIR is meant to be some nonstandard location containing the
  ; desired version of the tree-sitter library.
  ; The -rpath option tells the linker to hardcode this search location
  ; in the binary.
  ;
  ; This works as long as libtree-sitter stays where it is, which is
  ; fine for test executables. Production executables should instead
  ; link statically against libree-sitter to avoid problems in locating
  ; the library at runtime.
  ;
  (c_library_flags
    (
      -L%{env:TREESITTER_LIBDIR=/usr/local/lib}
      -lstdc++
      -ltree-sitter
      -Wl,-rpath,%{env:TREESITTER_LIBDIR=/usr/local/lib}
    )
  )
  (foreign_stubs
    (language cxx)
    (names ${cxx_files})
    (flags -fPIC
           -I %{env:TREESITTER_INCDIR=/usr/local/include}
           -I .)
  )
)
EOF

cat > "$dst_dir"/bin/dune <<EOF
(executable
  (package $package)
  (public_name parse-${lang_dashes})
  (name Main)
  (libraries $package.${lang_dashes})
)
EOF

# .gitignore file suitable for building with dune.
cat > "$dst_dir"/.gitignore <<EOF
# Dune build data
_build

# Merlin files for Vim and Emacs generated by dune < 2.8
.merlin

# Dune-generated files
*.install
EOF

############################################################################

# The following files are needed to build the libraries and executables
# as a standalone dune project. This is for local testing.
#
# We don't include these files in the destination git repo so it can be
# plugged into an existing git project without modifications.

cat > "$dst_dir"/dune-project <<EOF
(lang dune 3.7)
(name tree-sitter-$lang)
EOF

# This is needed for building with dune.
#
touch "$dst_dir"/tree-sitter-lang.opam