Merge branch develop to master

Author: xiangwang1

CHANGELOG.md

@@ -2,6 +2,18 @@
 
 This is a list of notable changes to Hyperscan, in reverse chronological order.
 
+## [5.0.0] 2018-07-09
+- Introduce chimera hybrid engine of Hyperscan and PCRE, to fully support
+  PCRE syntax as well as to take advantage of the high performance nature of
+  Hyperscan.
+- New API feature: logical combinations (AND, OR and NOT) of patterns in a
+  given pattern set.
+- Windows porting: hsbench, hscheck, hscollider and hsdump tools now available
+  on Windows 8 or newer.
+- Improve undirected graph implementation to avoid graph copy and reduce
+  compile time.
+- Bugfix for issue #86: enable hscollider for installed PCRE package.
+
 ## [4.7.0] 2018-01-24
 - Introduced hscollider pattern testing tool, for validating Hyperscan match
   behaviour against PCRE.

CMakeLists.txt

@@ -1,8 +1,8 @@
 cmake_minimum_required (VERSION 2.8.11)
 project (hyperscan C CXX)
 
-set (HS_MAJOR_VERSION 4)
-set (HS_MINOR_VERSION 7)
+set (HS_MAJOR_VERSION 5)
+set (HS_MINOR_VERSION 0)
 set (HS_PATCH_VERSION 0)
 set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION})
 
@@ -154,7 +154,7 @@ if(MSVC OR MSVC_IDE)
             # todo: change these as required
             set(ARCH_C_FLAGS "/arch:AVX2")
             set(ARCH_CXX_FLAGS "/arch:AVX2")
-            set(MSVC_WARNS "/wd4101 /wd4146 /wd4172 /wd4200 /wd4244 /wd4267 /wd4307 /wd4334 /wd4805 -D_CRT_SECURE_NO_WARNINGS")
+            set(MSVC_WARNS "/wd4101 /wd4146 /wd4172 /wd4200 /wd4244 /wd4267 /wd4307 /wd4334 /wd4805 /wd4996 -D_CRT_SECURE_NO_WARNINGS")
             set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}  /O2 ${MSVC_WARNS}")
             set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 ${MSVC_WARNS} /wd4800 -DBOOST_DETAIL_NO_CONTAINER_FWD")
         endif()
@@ -446,11 +446,32 @@ else()
 endif()
 
 add_subdirectory(util)
-add_subdirectory(unit)
 add_subdirectory(doc/dev-reference)
+
+if (NOT WIN32)
+# PCRE check, we have a fixed requirement for PCRE to use Chimera
+# and hscollider
+set(PCRE_REQUIRED_MAJOR_VERSION 8)
+set(PCRE_REQUIRED_MINOR_VERSION 41)
+set(PCRE_REQUIRED_VERSION ${PCRE_REQUIRED_MAJOR_VERSION}.${PCRE_REQUIRED_MINOR_VERSION})
+include (${CMAKE_MODULE_PATH}/pcre.cmake)
+if (NOT CORRECT_PCRE_VERSION)
+    message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} not found")
+endif()
+
+# we need static libs for Chimera - too much deep magic for shared libs
+if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS)
+    set(BUILD_CHIMERA TRUE)
+endif()
+
+add_subdirectory(unit)
 if (EXISTS ${CMAKE_SOURCE_DIR}/tools/CMakeLists.txt)
     add_subdirectory(tools)
 endif()
+if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA)
+    add_subdirectory(chimera)
+endif()
+endif()
 
 # do substitutions
 configure_file(${CMAKE_MODULE_PATH}/config.h.in ${PROJECT_BINARY_DIR}/config.h)
@@ -479,6 +500,31 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXX_FLAGS}")
 endif()
 
+if (WIN32)
+# PCRE check, we have a fixed requirement for PCRE to use Chimera
+# and hscollider
+set(PCRE_REQUIRED_MAJOR_VERSION 8)
+set(PCRE_REQUIRED_MINOR_VERSION 41)
+set(PCRE_REQUIRED_VERSION ${PCRE_REQUIRED_MAJOR_VERSION}.${PCRE_REQUIRED_MINOR_VERSION})
+include (${CMAKE_MODULE_PATH}/pcre.cmake)
+if (NOT CORRECT_PCRE_VERSION)
+    message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} not found")
+endif()
+
+# we need static libs for Chimera - too much deep magic for shared libs
+if (CORRECT_PCRE_VERSION AND PCRE_BUILD_SOURCE AND BUILD_STATIC_LIBS)
+    set(BUILD_CHIMERA TRUE)
+endif()
+
+add_subdirectory(unit)
+if (EXISTS ${CMAKE_SOURCE_DIR}/tools/CMakeLists.txt)
+    add_subdirectory(tools)
+endif()
+if (EXISTS ${CMAKE_SOURCE_DIR}/chimera/CMakeLists.txt AND BUILD_CHIMERA)
+    add_subdirectory(chimera)
+endif()
+endif()
+
 if(NOT WIN32)
 set(RAGEL_C_FLAGS "-Wno-unused")
 endif()
@@ -860,7 +906,6 @@ SET (hs_compile_SRCS
     src/nfagraph/ng_stop.h
     src/nfagraph/ng_uncalc_components.cpp
     src/nfagraph/ng_uncalc_components.h
-    src/nfagraph/ng_undirected.h
     src/nfagraph/ng_utf8.cpp
     src/nfagraph/ng_utf8.h
     src/nfagraph/ng_util.cpp
@@ -915,6 +960,8 @@ SET (hs_compile_SRCS
     src/parser/check_refs.h
     src/parser/control_verbs.cpp
     src/parser/control_verbs.h
+    src/parser/logical_combination.cpp
+    src/parser/logical_combination.h
     src/parser/parse_error.cpp
     src/parser/parse_error.h
     src/parser/parser_util.cpp
@@ -1014,6 +1061,7 @@ SET (hs_compile_SRCS
     src/util/graph.h
     src/util/graph_range.h
     src/util/graph_small_color_map.h
+    src/util/graph_undirected.h
     src/util/hash.h
     src/util/hash_dynamic_bitset.h
     src/util/insertion_ordered.h

chimera/CMakeLists.txt

@@ -0,0 +1,49 @@
+# Chimera lib
+
+include_directories(${PCRE_INCLUDE_DIRS})
+
+# only set these after all tests are done
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXX_FLAGS}")
+
+SET(chimera_HEADERS
+    ch.h
+    ch_common.h
+    ch_compile.h
+    ch_runtime.h
+)
+install(FILES ${chimera_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/hs")
+
+SET(chimera_SRCS
+    ${chimera_HEADERS}
+    ch_alloc.c
+    ch_alloc.h
+    ch_compile.cpp
+    ch_database.c
+    ch_database.h
+    ch_internal.h
+    ch_runtime.c
+    ch_scratch.h
+    ch_scratch.c
+)
+
+add_library(chimera STATIC ${chimera_SRCS})
+add_dependencies(chimera hs pcre)
+target_link_libraries(chimera hs pcre)
+
+install(TARGETS chimera DESTINATION ${CMAKE_INSTALL_LIBDIR})
+
+if (NOT WIN32)
+    # expand out library names for pkgconfig static link info
+    foreach (LIB ${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES})
+        # this is fragile, but protects us from toolchain specific files
+        if (NOT EXISTS ${LIB})
+            set(PRIVATE_LIBS "${PRIVATE_LIBS} -l${LIB}")
+        endif()
+    endforeach()
+    set(PRIVATE_LIBS "${PRIVATE_LIBS} -L${LIBDIR} -lpcre")
+
+    configure_file(libch.pc.in libch.pc @ONLY) # only replace @ quoted vars
+    install(FILES ${CMAKE_BINARY_DIR}/chimera/libch.pc
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+endif()

chimera/ch.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef CH_H_
+#define CH_H_
+
+/**
+ * @file
+ * @brief The complete Chimera API definition.
+ *
+ * Chimera is a hybrid solution of Hyperscan and PCRE.
+ *
+ * This header includes both the Chimera compiler and runtime components. See
+ * the individual component headers for documentation.
+ */
+
+#include "ch_compile.h"
+#include "ch_runtime.h"
+
+#endif /* CH_H_ */

chimera/ch_alloc.c

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2018, Intel Corporation
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** \file
+ * \brief Runtime functions for setting custom allocators.
+ */
+
+#include "ch.h"
+#include "ch_common.h"
+#include "ch_internal.h"
+#include "hs.h"
+#include "ue2common.h"
+
+#define default_malloc malloc
+#define default_free free
+
+ch_alloc_t ch_database_alloc = default_malloc;
+ch_alloc_t ch_misc_alloc = default_malloc;
+ch_alloc_t ch_scratch_alloc = default_malloc;
+
+ch_free_t ch_database_free = default_free;
+ch_free_t ch_misc_free = default_free;
+ch_free_t ch_scratch_free = default_free;
+
+static
+ch_alloc_t normalise_alloc(ch_alloc_t a) {
+    if (!a) {
+        return default_malloc;
+    } else {
+        return a;
+    }
+}
+
+static
+ch_free_t normalise_free(ch_free_t f) {
+    if (!f) {
+        return default_free;
+    } else {
+        return f;
+    }
+}
+
+HS_PUBLIC_API
+ch_error_t HS_CDECL ch_set_allocator(ch_alloc_t allocfunc,
+                                     ch_free_t freefunc) {
+    ch_set_database_allocator(allocfunc, freefunc);
+    ch_set_misc_allocator(allocfunc, freefunc);
+    ch_set_scratch_allocator(allocfunc, freefunc);
+
+    // Set core Hyperscan alloc/free.
+    hs_error_t ret = hs_set_allocator(allocfunc, freefunc);
+
+    return ret;
+}
+
+HS_PUBLIC_API
+ch_error_t HS_CDECL ch_set_database_allocator(ch_alloc_t allocfunc,
+                                              ch_free_t freefunc) {
+    ch_database_alloc = normalise_alloc(allocfunc);
+    ch_database_free = normalise_free(freefunc);
+
+    // Set Hyperscan database alloc/free.
+    return hs_set_database_allocator(allocfunc, freefunc);
+}
+
+HS_PUBLIC_API
+ch_error_t HS_CDECL ch_set_misc_allocator(ch_alloc_t allocfunc,
+                                          ch_free_t freefunc) {
+    ch_misc_alloc = normalise_alloc(allocfunc);
+    ch_misc_free = normalise_free(freefunc);
+
+    // Set Hyperscan misc alloc/free.
+    return hs_set_misc_allocator(allocfunc, freefunc);
+}
+
+HS_PUBLIC_API
+ch_error_t HS_CDECL ch_set_scratch_allocator(ch_alloc_t allocfunc,
+                                             ch_free_t freefunc) {
+    ch_scratch_alloc = normalise_alloc(allocfunc);
+    ch_scratch_free = normalise_free(freefunc);
+
+    // Set Hyperscan scratch alloc/free.
+    return hs_set_scratch_allocator(allocfunc, freefunc);
+}