Added unicode generators [PR was resubmitted]

include/proper.hrl

@@ -46,7 +46,8 @@
 -import(proper_types, [integer/2, float/2, atom/0, binary/0, binary/1,
 		       bitstring/0, bitstring/1, list/1, vector/2, union/1,
 		       weighted_union/1, tuple/1, loose_tuple/1, exactly/1,
-		       fixed_list/1, function/2, any/0]).
+		       fixed_list/1, function/2, any/0, 
+              improper_list/2, maybe_improper_list/2]).
 
 
 %%------------------------------------------------------------------------------
@@ -89,6 +90,15 @@
 			command_names/1, zip/2, run_parallel_commands/2,
 			run_parallel_commands/3]).
 
+
+%%------------------------------------------------------------------------------
+%% Unicode functions
+%%------------------------------------------------------------------------------
+
+-import(proper_unicode, [unicode_char/0, unicode_string/0, unicode_string/1,
+			unicode_binary/0, unicode_binary/1, unicode_binary/2, 
+            unicode_characters/0, unicode_characters/1]).
+
 -endif.
 
 

src/proper_types.erl

@@ -160,6 +160,7 @@
 -export([le/2]).
 
 -export_type([type/0, raw_type/0, extint/0, extnum/0]).
+-export([improper_list/2, maybe_improper_list/2]).
 
 -include("proper_internal.hrl").
 
@@ -189,7 +190,6 @@
 %% any:
 %%	doesn't cover functions and improper lists
 
-
 %%------------------------------------------------------------------------------
 %% Type declaration macros
 %%------------------------------------------------------------------------------
@@ -1110,6 +1110,13 @@ any() ->
 	{generator, fun proper_gen:any_gen/1}
     ]).
 
+-spec improper_list(T, T) -> T when T :: proper_types:type().
+improper_list(T,S) ->
+    ?LET({Contents, Terminator}, {list(T), S}, Contents ++ Terminator).
+
+-spec maybe_improper_list(T, T) -> T when T :: proper_types:type().
+maybe_improper_list(T, S) ->
+    union([list(T), improper_list(T, S)]).
 
 %%------------------------------------------------------------------------------
 %% Type aliases

src/proper_unicode.erl

@@ -0,0 +1,119 @@
+%% @doc This module provides basic unicode generators.
+%% @end
+%%
+%% @author Uvarov Michael <[email protected]>
+-module(proper_unicode).
+-export([unicode_char/0,
+         unicode_string/0,
+         unicode_string/1,
+         unicode_binary/0,
+         unicode_binary/1,
+         unicode_binary/2,
+         unicode_characters/0,
+         unicode_characters/1]).
+
+-define(PROPER_NO_IMPORTS, true).
+-import(proper_types, [char/0, list/1, vector/2,  
+                       maybe_improper_list/2, resize/2, frequency/1]).
+-include_lib("proper/include/proper.hrl").
+
+
+-type generator() :: term().
+
+-spec unicode_char() -> generator().
+unicode_char() ->
+    ?SIZED(Size, begin
+%   io:format(user, "Call ~p~n", [Size]),
+    frequency([{10, low_char()},
+               {4,  high_char()},
+               {2,  max_char()}]) end).
+
+low_char() ->
+    proper_char().
+
+high_char() ->
+    ?SIZED(Size, resize(Size * Size, proper_char())).
+
+max_char() ->
+    ?SIZED(Size, resize(Size * Size * Size * Size, proper_char())).
+
+proper_char() ->
+    ?SUCHTHAT(C, char(), (C < 16#D800 orelse C > 16#DFFF) andalso C =/= 16#FFFF 
+              andalso C =/= 16#FFFE).
+
+
+%% @doc Generate a list of unicode code points.
+-spec unicode_string() -> generator().
+unicode_string() ->
+    list(unicode_char()).
+
+
+%% @doc Generate a list of unicode code points of length `Size'.
+-spec unicode_string(non_neg_integer()) -> generator().
+
+unicode_string(Size) ->
+    vector(Size, unicode_char()).
+
+
+%% @doc Generate an unicode binary.
+-spec unicode_binary() -> generator().
+unicode_binary() ->
+    unicode_binary(undefined, unicode).
+
+
+%% @doc Generate an unicode binary binary.
+-spec unicode_binary(Size | Encoding) -> generator() when
+    Size :: non_neg_integer(),
+    Encoding :: unicode:encoding().
+
+unicode_binary(Size) when is_integer(Size) ->
+    unicode_binary(Size, unicode);
+
+unicode_binary(Encoding) ->
+    unicode_binary(undefined, Encoding).
+
+
+%% @doc Generate an unicode binary.
+-spec unicode_binary(Size, Encoding) -> generator() when
+    Size :: non_neg_integer() | undefined,
+    Encoding :: unicode:encoding().
+
+unicode_binary(undefined, Encoding) ->
+    ?LET(Str, unicode_string(),
+         unicode:characters_to_binary(Str, unicode, Encoding));
+
+unicode_binary(Size, Encoding) ->
+    ?LET(Str, unicode_string(Size),
+         unicode:characters_to_binary(Str, unicode, Encoding)).
+
+
+-spec unicode_characters() -> generator().
+
+unicode_characters() ->
+    unicode_characters(unicode).
+
+
+%% `unicode_characters()' should not return a single `unicode_char()'.
+-spec unicode_characters(Encoding) -> generator() when
+    Encoding :: unicode:encoding().
+
+unicode_characters(Encoding) ->
+    ?SIZED(Size,
+           frequency([{1, unicode_string()}, 
+                      {1, unicode_binary(Encoding)}, 
+                      {5, ?LAZY(resize(Size div 2, 
+                                       unicode_characters1(Encoding)))}
+                     ])).
+
+
+unicode_characters1(Encoding) ->
+    ?SIZED(Size, unicode_characters1(Size, Encoding)).
+
+
+unicode_characters1(0, _Encoding) ->
+    list(unicode_char());
+unicode_characters1(Size, Encoding) ->
+    Chars = ?LAZY(resize(Size, unicode_characters(Encoding))),
+    maybe_improper_list(frequency([{10,unicode_char()}, {1, Chars}]),
+                        unicode_binary(Encoding)).
+

test/proper_unicode_tests.erl

@@ -0,0 +1,76 @@
+%% @author Uvarov Michael <[email protected]>
+-module(proper_unicode_tests).
+
+%% ------------------------------------------------------------------
+%% Tests
+%% ------------------------------------------------------------------
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+
+%% ------------------------------------------------------------------
+%% Call test generators
+%% ------------------------------------------------------------------
+
+prop_unicode_char() ->
+    ?FORALL(Char, unicode_char(),
+        begin
+%           io:format(user, "~p~n", [Char]),
+            true
+        end).
+
+prop_unicode_binary() ->
+    ?FORALL(Bin, unicode_binary(),
+        begin
+        equals(Bin, unicode:characters_to_binary(
+                    unicode:characters_to_list(Bin)))
+        end).
+
+
+%% Check a binary generator with fixed length.
+prop_sized_unicode_binary() ->
+    ?FORALL({Len, Bin}, ?LET(Len, byte(), {Len, unicode_binary(Len)}),
+            equals(Len, length(unicode:characters_to_list(Bin)))).
+
+
+%% Check, that the `characters_to_list/1' does not fail.
+prop_unicode_string() ->
+    ?FORALL(Str, unicode_string(),
+            equals(Str, unicode:characters_to_list(
+                           unicode:characters_to_binary(Str)))).
+
+
+prop_unicode_characters() ->
+    ?FORALL(Chars, unicode_characters(),
+            is_binary(unicode:characters_to_binary(Chars))).
+
+
+encoding() ->
+    [unicode, utf8, utf16, {utf16, little}, {utf16, big}, utf32, 
+     {utf32, little}, {utf32, big}].
+
+
+prop_unicode_external_characters() ->
+    ?FORALL({Encoding, Chars}, 
+         oneof([{Encoding, unicode_characters(Encoding)} 
+                || Encoding <- encoding()]),
+            begin
+                List = unicode:characters_to_list(Chars, Encoding),
+                is_binary(unicode:characters_to_binary(Chars, Encoding))
+            end).
+
+
+%% -------------------------------------------------------------------
+%% Property Testing
+%% -------------------------------------------------------------------
+
+run_property_testing_test_() ->
+    {timeout, 60, fun run_property_testing_case/0}.
+
+run_property_testing_case() ->
+    EunitLeader = erlang:group_leader(),
+    erlang:group_leader(whereis(user), self()),
+    Res = proper:module(?MODULE, []),
+    erlang:group_leader(EunitLeader, self()),
+    ?assertEqual([], Res).