compiler: Add compile:string/1,2 for compiling source code strings

Add compile:string/1,2 and compile:noenv_string/2 that compile Erlang
source code from a string or binary. Unlike compile:forms, the source
goes through the Erlang preprocessor (epp), supporting -define, -include,
-ifdef, records, and all other preprocessor directives.

The `{include_path_open, Fun}` option is passed through to epp, enabling
fully in-memory compilation without file system access for include files.

Author: garazdawi

lib/compiler/src/compile.erl

@@ -208,6 +208,7 @@ source code.
 %% High-level interface.
 -export([file/1,file/2,noenv_file/2,format_error/1]).
 -export([forms/1,forms/2,noenv_forms/2]).
+-export([string/1,string/2,noenv_string/2]).
 -export([output_generated/1,noenv_output_generated/1]).
 -export([options/0]).
 -export([env_compiler_options/0]).
@@ -1047,6 +1048,51 @@ noenv_forms(Forms, Opts) when is_list(Opts) ->
 noenv_forms(Forms, Opt) when is_atom(Opt) ->
     noenv_forms(Forms, [Opt|?DEFAULT_OPTIONS]).
 
+-doc #{ equiv => string(String, []) }.
+-spec string(String :: unicode:chardata()) -> CompRet :: comp_ret().
+
+string(String) -> string(String, ?DEFAULT_OPTIONS).
+
+-doc """
+Compiles an Erlang source code string.
+
+Analogous to [`file/1`](`file/1`), but takes a `t:unicode:chardata/0` containing
+Erlang source code as first argument. The source code is run through the
+Erlang preprocessor (`epp`) just as when compiling a file, so directives
+such as `-include`, `-define`, and `-ifdef` are supported.
+
+Option `binary` is implicit, that is, no object code file is produced.
+
+The `source` option can be used to set the source file name that will
+appear in error messages and the `module_info/1` function. If not given,
+it defaults to the module name with a `.erl` extension.
+
+The `{include_path_open, Fun}` option can be used to provide a custom function
+for opening include files (see `epp:open/1`), allowing compilation
+entirely in memory without touching the file system.
+
+```erlang
+1> compile:string("-module(foo). -export([bar/0]). bar() -> ok.").
+{ok,foo,<<...>>}
+```
+""".
+-spec string(String :: unicode:chardata(), Options :: [option()] | option()) ->
+          CompRet :: comp_ret().
+
+string(String, Opts) when is_list(Opts) ->
+    do_compile({string,String}, [binary|Opts++env_default_opts()]);
+string(String, Opt) when is_atom(Opt) ->
+    string(String, [Opt|?DEFAULT_OPTIONS]).
+
+-doc #{ equiv => noenv_string(String, []) }.
+-spec noenv_string(String :: unicode:chardata(), Options :: [option()] | option()) ->
+          comp_ret().
+
+noenv_string(String, Opts) when is_list(Opts) ->
+    do_compile({string,String}, [binary|Opts]);
+noenv_string(String, Opt) when is_atom(Opt) ->
+    noenv_string(String, [Opt|?DEFAULT_OPTIONS]).
+
 -doc """
 Works like `output_generated/1`, except that the environment variable
 `ERL_COMPILER_OPTIONS` is not consulted.
@@ -1301,6 +1347,14 @@ internal({forms,Forms}, Opts0) ->
                        strip_columns(Forms)
                end,
     internal_comp(Ps, NewForms, Source, "", Compile);
+internal({string,String}, Opts0) ->
+    Bin = unicode:characters_to_binary(String),
+    {ok, Fd} = file:open(Bin, [ram, read, binary, cooked]),
+    try
+        do_parse_string(Fd, Opts0)
+    after
+        file:close(Fd)
+    end;
 internal({file,File}, Opts) ->
     {Ext,Ps} = passes(file, Opts),
     Compile = build_compile(Opts),
@@ -2004,6 +2058,71 @@ parse_module(_Code, St) ->
 	    Ret
     end.
 
+do_parse_string(Fd, Opts0) ->
+    StartLocation = case with_columns(Opts0) of
+                        true ->
+                            {1,1};
+                        false ->
+                            1
+                    end,
+    case erl_features:init_parse_state(Opts0, fun erl_scan:f_reserved_word/1) of
+        {ok, {Features, ResWordFun}} ->
+            PathOpenOpt = case proplists:get_value(include_path_open, Opts0) of
+                              undefined -> [];
+                              PathOpenFun -> [{include_path_open, PathOpenFun}]
+                          end,
+            Name = proplists:get_value(source, Opts0, "string"),
+            EppOpts = [{fd, Fd},
+                       {name, Name},
+                       {includes, ["." | inc_paths(Opts0)]},
+                       {macros, pre_defs(Opts0)},
+                       {default_encoding, utf8},
+                       {location, StartLocation},
+                       {reserved_word_fun, ResWordFun},
+                       {features, Features},
+                       extra |
+                       PathOpenOpt ++
+                       case member(check_ssa, Opts0) of
+                           true ->
+                               [{compiler_internal, [ssa_checks]}];
+                           false ->
+                               []
+                       end],
+            {ok, Epp, Extra0} = epp:open(EppOpts),
+            try
+                Forms0 = epp:parse_file(Epp),
+                Epp ! {get_features, self()},
+                UsedFtrs = receive {features, X} -> X end,
+                Extra = [{features, UsedFtrs} | Extra0],
+                Encoding = proplists:get_value(encoding, Extra),
+                {_, Ps} = passes(forms, Opts0),
+                Source = proplists:get_value(source, Opts0, source_from_forms(Forms0)),
+                Opts1 = proplists:delete(source, Opts0),
+                Compile0 = build_compile(Opts1),
+                St0 = metadata_add_features(UsedFtrs, Compile0),
+                Opts2 = [{features, UsedFtrs} | St0#compile.options],
+                St1 = St0#compile{encoding=Encoding, options=Opts2},
+                Forms = case with_columns(Opts2 ++ compile_options(Forms0)) of
+                            true ->
+                                Forms0;
+                            false ->
+                                strip_columns(Forms0)
+                        end,
+                internal_comp(Ps, Forms, Source, "", St1)
+            after
+                epp:close(Epp)
+            end;
+        {error, {Mod, Reason}} ->
+            Source = proplists:get_value(source, Opts0, "string"),
+            Es = [{Source, [{none, Mod, Reason}]}],
+            {error, {errors, Es, []}}
+    end.
+
+source_from_forms([{attribute,_,module,Mod}|_]) ->
+    atom_to_list(Mod) ++ ".erl";
+source_from_forms([_|T]) -> source_from_forms(T);
+source_from_forms([]) -> "string".
+
 deterministic_filename(#compile{ifile=File,options=Opts}) ->
     SourceName0 = proplists:get_value(source, Opts, File),
     case member(deterministic, Opts) of

lib/compiler/test/compile_SUITE.erl

@@ -27,11 +27,11 @@
 -include_lib("stdlib/include/erl_compile.hrl").
 -include_lib("stdlib/include/assert.hrl").
 
--export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 
+-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1,
 	 init_per_group/2,end_per_group/2,
 	 app_test/1,appup_test/1,bigE_roundtrip/1,
 	 debug_info/4, custom_debug_info/1, custom_compile_info/1,
-	 file_1/1, forms_2/1, module_mismatch/1, outdir/1,
+         file_1/1, forms_2/1, string_1/1, module_mismatch/1, outdir/1,
 	 binary/1, makedep/1, cond_and_ifdef/1, listings/1, listings_big/1,
 	 other_output/1, encrypted_abstr/1,
 	 strict_record/1, utf8_atoms/1, utf8_functions/1, extra_chunks/1,
@@ -60,7 +60,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
     [app_test, appup_test, bigE_roundtrip, file_1,
-     forms_2, module_mismatch, outdir,
+     forms_2, string_1, module_mismatch, outdir,
      binary, makedep, cond_and_ifdef, listings, listings_big,
      other_output, encrypted_abstr, tuple_calls,
      strict_record, utf8_atoms, utf8_functions, extra_chunks,
@@ -326,6 +326,169 @@ forms_compile_and_load(Code, Opts) ->
     false = code:purge(simple),
     ok.
 
+string_1(_Config) ->
+    %% Basic compilation from string.
+    {ok,foo,BinFoo} = compile:string("-module(foo). -export([bar/0]). bar() -> hello."),
+    {module,foo} = code:load_binary(foo, "foo.erl", BinFoo),
+    hello = foo:bar(),
+    true = code:delete(foo),
+    false = code:purge(foo),
+
+    %% Binary input.
+    {ok,binmod,BinMod} = compile:string(<<"-module(binmod). -export([g/0]). g() -> ok.">>),
+    {module,binmod} = code:load_binary(binmod, "binmod.erl", BinMod),
+    ok = binmod:g(),
+    true = code:delete(binmod),
+    false = code:purge(binmod),
+
+    %% Preprocessor: macros with -define.
+    {ok,macmod,BinMac} = compile:string(
+        "-module(macmod). -export([f/0]).\n"
+        "-define(X, 42).\n"
+        "f() -> ?X.\n"),
+    {module,macmod} = code:load_binary(macmod, "macmod.erl", BinMac),
+    42 = macmod:f(),
+    true = code:delete(macmod),
+    false = code:purge(macmod),
+
+    %% Preprocessor: records.
+    {ok,recmod,BinRec} = compile:string(
+        "-module(recmod). -export([new/0]).\n"
+        "-record(point, {x = 0, y = 0}).\n"
+        "new() -> #point{x = 1, y = 2}.\n"),
+    {module,recmod} = code:load_binary(recmod, "recmod.erl", BinRec),
+    {point,1,2} = recmod:new(),
+    true = code:delete(recmod),
+    false = code:purge(recmod),
+
+    %% Preprocessor: -ifdef/-endif.
+    {ok,ifmod,BinIf} = compile:string(
+        "-module(ifmod). -export([f/0]).\n"
+        "-ifdef(NOTDEFINED).\n"
+        "f() -> wrong.\n"
+        "-else.\n"
+        "f() -> right.\n"
+        "-endif.\n"),
+    {module,ifmod} = code:load_binary(ifmod, "ifmod.erl", BinIf),
+    right = ifmod:f(),
+    true = code:delete(ifmod),
+    false = code:purge(ifmod),
+
+    %% Preprocessor: predefined macros via options.
+    {ok,defmod,BinDef} = compile:string(
+        "-module(defmod). -export([f/0]).\n"
+        "f() -> ?MY_VALUE.\n",
+        [{d,'MY_VALUE',99}]),
+    {module,defmod} = code:load_binary(defmod, "defmod.erl", BinDef),
+    99 = defmod:f(),
+    true = code:delete(defmod),
+    false = code:purge(defmod),
+
+    %% Preprocessor: ?MODULE.
+    {ok,modmac,BinModMac} = compile:string(
+        "-module(modmac). -export([name/0]).\n"
+        "name() -> ?MODULE.\n"),
+    {module,modmac} = code:load_binary(modmac, "modmac.erl", BinModMac),
+    modmac = modmac:name(),
+    true = code:delete(modmac),
+    false = code:purge(modmac),
+
+    %% Source option.
+    {ok,srcmod,BinSrc} = compile:string(
+        "-module(srcmod). -export([f/0]). f() -> ok.",
+        [{source,"my_source.erl"}]),
+    {module,srcmod} = code:load_binary(srcmod, "srcmod.erl", BinSrc),
+    Info = srcmod:module_info(compile),
+    SrcInfo = proplists:get_value(source, Info),
+    true = lists:suffix("my_source.erl", SrcInfo),
+    true = code:delete(srcmod),
+    false = code:purge(srcmod),
+
+    %% Syntax error returns error.
+    error = compile:string("-module(bad). f( ->"),
+
+    %% Syntax error with return_errors option.
+    {error,[{"string",[{_,_,_}|_]}],_} =
+        compile:string("-module(bad). f( ->", [return_errors]),
+
+    %% Source option affects error filenames.
+    {error,[{"bad.erl",[{_,_,_}|_]}],_} =
+        compile:string("-module(bad). f( ->",
+                       [return_errors, {source, "bad.erl"}]),
+
+    %% Cover: option not in a list (undocumented feature).
+    {ok,smod,_} = compile:string("-module(smod). -export([f/0]). f() -> ok.", binary),
+
+    %% noenv_string/2.
+    {ok,nmod,_} = compile:noenv_string(
+        "-module(nmod). -export([f/0]). f() -> ok.", []),
+
+    %% include_path_open: include from in-memory files.
+    Headers = #{
+        "my_header.hrl" =>
+            <<"-record(point, {x, y}).\n"
+              "-define(ORIGIN, #point{x=0, y=0}).\n">>
+    },
+    PathOpen = fun(_Path, Name, _Modes) ->
+        case maps:find(Name, Headers) of
+            {ok, Content} ->
+                {ok, Fd} = file:open(Content, [ram, read, binary, cooked]),
+                {ok, Fd, Name};
+            error ->
+                {error, enoent}
+        end
+    end,
+    {ok,incmod,BinInc} = compile:string(
+        "-module(incmod).\n"
+        "-include(\"my_header.hrl\").\n"
+        "-export([f/0]).\n"
+        "f() -> ?ORIGIN.\n",
+        [{include_path_open, PathOpen}]),
+    {module,incmod} = code:load_binary(incmod, "incmod.erl", BinInc),
+    {point,0,0} = incmod:f(),
+    true = code:delete(incmod),
+    false = code:purge(incmod),
+
+    %% include_path_open: nested includes from in-memory files.
+    Headers2 = #{
+        "types.hrl" =>
+            <<"-type my_int() :: integer().\n">>,
+        "all.hrl" =>
+            <<"-include(\"types.hrl\").\n"
+              "-define(DEFAULT, 0).\n">>
+    },
+    PathOpen2 = fun(_Path, Name, _Modes) ->
+        case maps:find(Name, Headers2) of
+            {ok, Content} ->
+                {ok, Fd} = file:open(Content, [ram, read, binary, cooked]),
+                {ok, Fd, Name};
+            error ->
+                {error, enoent}
+        end
+    end,
+    {ok,nestmod,BinNest} = compile:string(
+        "-module(nestmod).\n"
+        "-include(\"all.hrl\").\n"
+        "-export([f/0]).\n"
+        "-spec f() -> my_int().\n"
+        "f() -> ?DEFAULT.\n",
+        [{include_path_open, PathOpen2}]),
+    {module,nestmod} = code:load_binary(nestmod, "nestmod.erl", BinNest),
+    0 = nestmod:f(),
+    true = code:delete(nestmod),
+    false = code:purge(nestmod),
+
+    %% include_path_open: missing include returns error.
+    PathOpenNone = fun(_Path, _Name, _Modes) -> {error, enoent} end,
+    {error, _, _} = compile:string(
+        "-module(missinc).\n"
+        "-include(\"nonexistent.hrl\").\n"
+        "-export([f/0]).\n"
+        "f() -> ok.\n",
+        [{include_path_open, PathOpenNone}, return_errors]),
+
+    ok.
+
 module_mismatch(Config) when is_list(Config) ->
     DataDir = proplists:get_value(data_dir, Config),
     File = filename:join(DataDir, "wrong_module_name.erl"),