Skip to content

Commit 006d0ad

Browse files
committed
feat: fff for file search
This is how it works: https://github.com/user-attachments/assets/b9627617-b274-4a38-9ca3-13d6e7c8e963 A few important descisions/questions: 1. fff doesn't persist the index, the index is extremely fast so I don't see a reason to vaste resources storing paths anywhere. The index lives per the scope of open window triggered by file search view open 2. fff uses databases for access patterns, I suppose a lot of people would be happy if we reuse their existing database, I added a logic for this. Though your recently accessed files feature is separate 3. The search open window now shows the amount of indexed files in real time, can revert this - just thinking this is cool This PR was done with assistance of Claude Opus 4.7
1 parent 9c59057 commit 006d0ad

23 files changed

Lines changed: 1641 additions & 139 deletions

CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ if (NOT USE_SYSTEM_GLAZE)
6363
import_glaze()
6464
endif()
6565

66+
include(Fff)
67+
fff_configure()
68+
6669
if (NOT USE_SYSTEM_KF6)
6770
include(KF6)
6871
import_kf6()

cmake/CMark.cmake

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,23 @@ function(checkout_cmark)
1818

1919

2020
FetchContent_MakeAvailable(cmark-gfm)
21+
22+
# Upstream cmark-gfm uses bare `include_directories(.)` which does not
23+
# propagate to consumers. Expose the headers as INTERFACE paths so
24+
# anything linking `libcmark-gfm_static` / `libcmark-gfm-extensions_static`
25+
# can actually find `cmark-gfm.h` and the generated `cmark-gfm_export.h`.
26+
if (TARGET libcmark-gfm_static)
27+
target_include_directories(libcmark-gfm_static INTERFACE
28+
"${cmark-gfm_SOURCE_DIR}/src"
29+
"${cmark-gfm_BINARY_DIR}/src"
30+
"${cmark-gfm_BINARY_DIR}/extensions")
31+
endif()
32+
if (TARGET libcmark-gfm-extensions_static)
33+
target_include_directories(libcmark-gfm-extensions_static INTERFACE
34+
"${cmark-gfm_SOURCE_DIR}/extensions"
35+
"${cmark-gfm_SOURCE_DIR}/src"
36+
"${cmark-gfm_BINARY_DIR}/src"
37+
"${cmark-gfm_BINARY_DIR}/extensions")
38+
endif()
2139
set(CMAKE_SKIP_INSTALL_RULES OFF)
2240
endfunction()

cmake/Fff.cmake

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
# opts:
2+
# FFF_VERSION (string) - release tag / git ref.
3+
# FFF_LIBC (glibc|musl|auto) - only for prebuilt Linux.
4+
# FFF_BUILD_FROM_SOURCE (BOOL, default OFF) - cargo build fff locally.
5+
# FFF_CARGO_FEATURES (string, default "") - comma list for --features.
6+
# FFF_CARGO_PROFILE (release|dev, default release)
7+
#
8+
# outputs of ${CMAKE_BINARY_DIR}/_fff:
9+
# _fff/lib/libfff_c.<ext> (prebuilt mode)
10+
# _fff/include/fff.h (prebuilt mode)
11+
# _fff/.stamp-<FFF_VERSION>-<triple> (prebuilt cache invalidator)
12+
# _fff/src/ (source-build clone)
13+
# _fff/cargo/<profile>/libfff_c.<ext> (source-build output)
14+
15+
set(FFF_VERSION "v0.8.1" CACHE STRING "fff release tag / git ref to use")
16+
set(FFF_LIBC "auto" CACHE STRING "Linux C library variant for fff: glibc | musl | auto")
17+
set_property(CACHE FFF_LIBC PROPERTY STRINGS "auto" "glibc" "musl")
18+
option(FFF_BUILD_FROM_SOURCE "Build libfff_c locally with cargo (requires Rust toolchain)" OFF)
19+
set(FFF_CARGO_FEATURES "" CACHE STRING "fff feature flags, provide 'zlob' if you have zig toolchain installed")
20+
set(FFF_CARGO_PROFILE "release" CACHE STRING "Cargo profile for fff-c (release | dev)")
21+
set_property(CACHE FFF_CARGO_PROFILE PROPERTY STRINGS "release" "dev")
22+
23+
function(_fff_detect_libc out_libc)
24+
# probe ldd --version if output
25+
execute_process(
26+
COMMAND ldd --version
27+
OUTPUT_VARIABLE _ldd_out
28+
ERROR_VARIABLE _ldd_err
29+
TIMEOUT 5)
30+
31+
if ("${_ldd_out}${_ldd_err}" MATCHES "musl")
32+
set(${out_libc} "musl" PARENT_SCOPE)
33+
else()
34+
set(${out_libc} "glibc" PARENT_SCOPE)
35+
endif()
36+
endfunction()
37+
38+
function(_fff_detect_triple out_triple out_ext)
39+
# Normalize processor
40+
set(_proc "${CMAKE_SYSTEM_PROCESSOR}")
41+
if (_proc STREQUAL "AMD64" OR _proc STREQUAL "x64")
42+
set(_proc "x86_64")
43+
elseif (_proc STREQUAL "arm64")
44+
set(_proc "aarch64")
45+
endif()
46+
47+
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
48+
set(_libc "${FFF_LIBC}")
49+
if (_libc STREQUAL "auto")
50+
_fff_detect_libc(_libc)
51+
message(STATUS "fff: auto-detected libc=${_libc}")
52+
endif()
53+
54+
if (_libc STREQUAL "musl")
55+
set(${out_triple} "${_proc}-unknown-linux-musl" PARENT_SCOPE)
56+
else()
57+
set(${out_triple} "${_proc}-unknown-linux-gnu" PARENT_SCOPE)
58+
endif()
59+
set(${out_ext} "so" PARENT_SCOPE)
60+
elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") # is vicinae even compiled for macos? would be fun lol
61+
set(${out_triple} "${_proc}-apple-darwin" PARENT_SCOPE)
62+
set(${out_ext} "dylib" PARENT_SCOPE)
63+
else()
64+
message(FATAL_ERROR "fff does not publish a prebuilt C library for ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}. Build from source with -DFFF_BUILD_FROM_SOURCE=ON.")
65+
endif()
66+
endfunction()
67+
68+
function(_fff_download url dst)
69+
message(STATUS "fff: fetching ${url}")
70+
file(DOWNLOAD "${url}" "${dst}"
71+
TLS_VERIFY ON
72+
STATUS _status)
73+
list(GET _status 0 _rc)
74+
if (NOT _rc EQUAL 0)
75+
list(GET _status 1 _err)
76+
message(FATAL_ERROR "fff: failed to download ${url}: ${_err}")
77+
endif()
78+
endfunction()
79+
80+
function(_fff_configure_prebuilt)
81+
_fff_detect_triple(_triple _ext)
82+
83+
set(_root "${CMAKE_BINARY_DIR}/_fff")
84+
set(_lib_dir "${_root}/lib")
85+
set(_inc_dir "${_root}/include")
86+
set(_libfile "${_lib_dir}/libfff_c.${_ext}")
87+
set(_hdrfile "${_inc_dir}/fff.h")
88+
set(_stamp "${_root}/.stamp-${FFF_VERSION}-${_triple}")
89+
90+
if (NOT EXISTS "${_stamp}")
91+
# Version or triple changed. Wipe any prior prebuilt cache.
92+
if (EXISTS "${_lib_dir}")
93+
file(REMOVE_RECURSE "${_lib_dir}")
94+
endif()
95+
if (EXISTS "${_inc_dir}")
96+
file(REMOVE_RECURSE "${_inc_dir}")
97+
endif()
98+
file(GLOB _old_stamps "${_root}/.stamp-*")
99+
if (_old_stamps)
100+
file(REMOVE ${_old_stamps})
101+
endif()
102+
file(MAKE_DIRECTORY "${_lib_dir}" "${_inc_dir}")
103+
104+
set(_base "https://github.com/dmtrKovalenko/fff/releases/download/${FFF_VERSION}")
105+
set(_asset "c-lib-${_triple}.${_ext}")
106+
_fff_download("${_base}/${_asset}" "${_libfile}")
107+
_fff_download(
108+
"https://raw.githubusercontent.com/dmtrKovalenko/fff/${FFF_VERSION}/crates/fff-c/include/fff.h"
109+
"${_hdrfile}")
110+
111+
file(WRITE "${_stamp}" "${FFF_VERSION} ${_triple}\n")
112+
endif()
113+
114+
add_library(vicinae::fff SHARED IMPORTED GLOBAL)
115+
set_target_properties(vicinae::fff PROPERTIES
116+
IMPORTED_LOCATION "${_libfile}"
117+
IMPORTED_NO_SONAME TRUE
118+
INTERFACE_INCLUDE_DIRECTORIES "${_inc_dir}")
119+
120+
set(FFF_RUNTIME_LIBRARY "${_libfile}" CACHE INTERNAL "" FORCE)
121+
message(STATUS "fff: prebuilt ${FFF_VERSION} (${_triple}) at ${_libfile}")
122+
endfunction()
123+
124+
function(_fff_configure_source)
125+
find_program(CARGO cargo)
126+
if (NOT CARGO)
127+
message(FATAL_ERROR
128+
"fff: cargo not found but -DFFF_BUILD_FROM_SOURCE=ON was requested.\n"
129+
"Install the Rust toolchain (https://rustup.rs) or unset "
130+
"-DFFF_BUILD_FROM_SOURCE to use the prebuilt binary.")
131+
endif()
132+
133+
# Library extension still comes from the triple detector; we only use the
134+
# extension part for source builds.
135+
_fff_detect_triple(_ignore_triple _ext)
136+
137+
include(FetchContent)
138+
FetchContent_Declare(
139+
fff_src
140+
GIT_REPOSITORY https://github.com/dmtrKovalenko/fff.git
141+
GIT_TAG ${FFF_VERSION}
142+
GIT_SHALLOW TRUE
143+
EXCLUDE_FROM_ALL
144+
SOURCE_DIR "${CMAKE_BINARY_DIR}/_fff/src"
145+
)
146+
147+
# do not invoke subdir becuase fff is a rust project without cmake
148+
FetchContent_GetProperties(fff_src)
149+
if (NOT fff_src_POPULATED)
150+
message(STATUS "fff: cloning source tree ${FFF_VERSION}")
151+
# FetchContent_Populate is deprecated in 3.30+ but still the supported
152+
# way to populate without add_subdirectory. Quiet the warning.
153+
if (POLICY CMP0169)
154+
cmake_policy(PUSH)
155+
cmake_policy(SET CMP0169 OLD)
156+
endif()
157+
FetchContent_Populate(fff_src)
158+
if (POLICY CMP0169)
159+
cmake_policy(POP)
160+
endif()
161+
endif()
162+
163+
set(_src_dir "${fff_src_SOURCE_DIR}")
164+
set(_cargo_dir "${CMAKE_BINARY_DIR}/_fff/cargo")
165+
set(_profile_dir_name "${FFF_CARGO_PROFILE}")
166+
if (FFF_CARGO_PROFILE STREQUAL "dev")
167+
# cargo's `dev` profile outputs into `debug/`.
168+
set(_profile_dir_name "debug")
169+
endif()
170+
set(_libfile "${_cargo_dir}/${_profile_dir_name}/libfff_c.${_ext}")
171+
set(_hdrdir "${_src_dir}/crates/fff-c/include")
172+
173+
set(_cargo_args build -p fff-c
174+
--manifest-path "${_src_dir}/Cargo.toml"
175+
--target-dir "${_cargo_dir}")
176+
177+
if (FFF_CARGO_PROFILE STREQUAL "release")
178+
list(APPEND _cargo_args --release)
179+
elseif (NOT FFF_CARGO_PROFILE STREQUAL "dev")
180+
message(FATAL_ERROR "fff: FFF_CARGO_PROFILE must be 'release' or 'dev' (got '${FFF_CARGO_PROFILE}')")
181+
endif()
182+
183+
if (FFF_CARGO_FEATURES)
184+
string(REPLACE " " "," _features "${FFF_CARGO_FEATURES}")
185+
list(APPEND _cargo_args --features "${_features}")
186+
endif()
187+
188+
# cargo manages it's own compilation, so we invalicate it on version or rust code change
189+
file(GLOB_RECURSE _fff_src_glob
190+
CONFIGURE_DEPENDS
191+
"${_src_dir}/crates/fff-c/src/*.rs"
192+
"${_src_dir}/crates/fff-c/build.rs"
193+
"${_src_dir}/crates/fff-c/Cargo.toml")
194+
195+
add_custom_command(
196+
OUTPUT "${_libfile}"
197+
COMMAND ${CARGO} ${_cargo_args}
198+
WORKING_DIRECTORY "${_src_dir}"
199+
DEPENDS
200+
"${_src_dir}/Cargo.toml"
201+
"${_src_dir}/Cargo.lock"
202+
${_fff_src_glob}
203+
COMMENT "fff: cargo build (${FFF_CARGO_PROFILE}, features=${FFF_CARGO_FEATURES})"
204+
VERBATIM
205+
USES_TERMINAL)
206+
207+
add_custom_target(fff_c_build ALL DEPENDS "${_libfile}")
208+
209+
add_library(vicinae::fff SHARED IMPORTED GLOBAL)
210+
set_target_properties(vicinae::fff PROPERTIES
211+
IMPORTED_LOCATION "${_libfile}"
212+
IMPORTED_NO_SONAME TRUE
213+
INTERFACE_INCLUDE_DIRECTORIES "${_hdrdir}")
214+
add_dependencies(vicinae::fff fff_c_build)
215+
216+
set(FFF_RUNTIME_LIBRARY "${_libfile}" CACHE INTERNAL "" FORCE)
217+
message(STATUS "fff: building from source (${FFF_VERSION}, profile=${FFF_CARGO_PROFILE}, features=${FFF_CARGO_FEATURES}) -> ${_libfile}")
218+
endfunction()
219+
220+
function(fff_configure)
221+
if (TARGET vicinae::fff)
222+
return()
223+
endif()
224+
225+
if (FFF_BUILD_FROM_SOURCE)
226+
_fff_configure_source()
227+
else()
228+
_fff_configure_prebuilt()
229+
endif()
230+
endfunction()

src/server/CMakeLists.txt

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ list(APPEND LIBS
1414
Qt6::Sql Qt6::Network Qt6::Svg Qt6::DBus Qt6::Concurrent Qt6::Quick Qt6::Qml
1515
Qt6::GuiPrivate # for deeper integration with wayland protocols, we need wl_surface
1616
Qt6::QuickDialogs2 Qt6::QuickControls2
17-
${CMARK_LIBRARY} ${CMARK_EXT_LIBRARY}
17+
${CMARK_EXT_LIBRARY} ${CMARK_LIBRARY}
1818
minizip
1919
OpenSSL::Crypto
2020
wayland-client
@@ -27,6 +27,8 @@ list(APPEND LIBS
2727
vicinae::emoji
2828
vicinae::common
2929
vicinae::fuzzy
30+
vicinae::linuxutils
31+
vicinae::fff
3032
)
3133

3234

@@ -368,6 +370,11 @@ set(SRCS
368370
src/services/files-service/file-indexer/abstract-scanner.hpp
369371
src/services/files-service/file-indexer/file-indexer-query-engine.hpp
370372

373+
src/services/files-service/fff/fff-library.hpp
374+
src/services/files-service/fff/fff-library.cpp
375+
src/services/files-service/fff/fff-file-indexer.hpp
376+
src/services/files-service/fff/fff-file-indexer.cpp
377+
371378
src/services/extension-registry/extension-registry.hpp
372379
src/services/extension-registry/extension-registry.cpp
373380
src/services/extension-registry/extension-manifest.hpp
@@ -891,6 +898,30 @@ install(TARGETS ${TARGET}
891898
RUNTIME DESTINATION ${VICINAE_LIBEXEC_DIR}
892899
)
893900

901+
# Ship libfff_c.so next to vicinae-server and make the linker find it at
902+
# runtime through a relative rpath. $ORIGIN resolves to the directory of the
903+
# server binary at execution time.
904+
if (FFF_RUNTIME_LIBRARY)
905+
install(FILES "${FFF_RUNTIME_LIBRARY}"
906+
DESTINATION ${VICINAE_LIBEXEC_DIR}
907+
PERMISSIONS
908+
OWNER_READ OWNER_WRITE OWNER_EXECUTE
909+
GROUP_READ GROUP_EXECUTE
910+
WORLD_READ WORLD_EXECUTE)
911+
912+
if (APPLE)
913+
set(_fff_rpath "@loader_path")
914+
else()
915+
set(_fff_rpath "$ORIGIN")
916+
endif()
917+
918+
set_target_properties(${TARGET} PROPERTIES
919+
BUILD_WITH_INSTALL_RPATH FALSE
920+
BUILD_RPATH "${CMAKE_BINARY_DIR}/_fff/lib"
921+
INSTALL_RPATH "${_fff_rpath}"
922+
INSTALL_RPATH_USE_LINK_PATH TRUE)
923+
endif()
924+
894925
if (BUILD_TESTS)
895926
set(TEST_TARGET ${TARGET}-tests)
896927
find_package(Catch2 3 REQUIRED)
@@ -903,3 +934,41 @@ if (BUILD_TESTS)
903934
target_link_libraries(${TEST_TARGET} PRIVATE Catch2::Catch2WithMain Qt6::Core Qt6::Gui)
904935
target_compile_features(${TEST_TARGET} PUBLIC cxx_std_23)
905936
endif()
937+
938+
# Standalone smoke test for the fff integration. Not gated on BUILD_TESTS,
939+
# because it does not require Catch2 - builds opt-in via -DBUILD_FFF_SMOKE=ON.
940+
option(BUILD_FFF_SMOKE "Build the fff integration smoke test binary" OFF)
941+
if (BUILD_FFF_SMOKE)
942+
set(FFF_SMOKE_TARGET ${TARGET}-fff-smoke)
943+
add_executable(${FFF_SMOKE_TARGET}
944+
tests/fff-smoke.cpp
945+
src/services/files-service/abstract-file-indexer.hpp
946+
src/services/files-service/fff/fff-library.cpp
947+
src/services/files-service/fff/fff-file-indexer.hpp
948+
src/services/files-service/fff/fff-file-indexer.cpp
949+
src/utils/utils.cpp
950+
src/vicinae.cpp
951+
)
952+
set_target_properties(${FFF_SMOKE_TARGET} PROPERTIES AUTOMOC ON)
953+
target_include_directories(${FFF_SMOKE_TARGET} PRIVATE
954+
${CMAKE_CURRENT_SOURCE_DIR}/src
955+
${CMAKE_CURRENT_SOURCE_DIR}/src/lib
956+
${CMAKE_CURRENT_SOURCE_DIR}/src/utils
957+
${CMAKE_CURRENT_SOURCE_DIR}/..
958+
${CMAKE_CURRENT_BINARY_DIR})
959+
target_link_libraries(${FFF_SMOKE_TARGET} PRIVATE
960+
Qt6::Core Qt6::Concurrent Qt6::Gui
961+
glaze::glaze
962+
vicinae::fff
963+
vicinae::xdgpp)
964+
target_compile_features(${FFF_SMOKE_TARGET} PUBLIC cxx_std_26)
965+
966+
if (APPLE)
967+
set(_fff_smoke_rpath "@loader_path")
968+
else()
969+
set(_fff_smoke_rpath "$ORIGIN")
970+
endif()
971+
set_target_properties(${FFF_SMOKE_TARGET} PROPERTIES
972+
BUILD_RPATH "${CMAKE_BINARY_DIR}/_fff/lib"
973+
INSTALL_RPATH "${_fff_smoke_rpath}")
974+
endif()

src/server/src/extensions/file/file-extension.hpp

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,11 @@ class FileExtension : public BuiltinCommandRepository {
6262

6363
public:
6464
void initialized(const QJsonObject &preferences) const override {
65-
auto files = ServiceRegistry::instance()->fileService();
66-
if (preferences.value("autoIndexing").toBool()) { files->indexer()->start(); }
65+
// TODO check this out on review: I do not think that it makes sense to do any file system
66+
// scanning in the background, on ext4 wired with getdents64 + the default garbage filters
67+
// fff has (e.g. skipping node modules, targets, and binary files) 1TB files would be indexed
68+
// within a few seconds, so I do not think there is a reason to keep all of those in memory
69+
Q_UNUSED(preferences);
6770
}
6871

6972
FileExtension() {
@@ -97,7 +100,15 @@ class FileExtension : public BuiltinCommandRepository {
97100
watcherPaths.setDescription("Semicolon-separated list of paths watched by experimental watcher");
98101
watcherPaths.setDefaultValue("");
99102

100-
return {indexing, paths, excludedPaths, watcherPaths};
103+
auto reuseNvim = Preference::makeCheckbox("reuseNvimDbs");
104+
reuseNvim.setTitle("Reuse fff.nvim databases");
105+
reuseNvim.setDescription(
106+
"When enabled and the fff.nvim plugin's frecency / history databases are detected at "
107+
"$XDG_CACHE_HOME/nvim/fff_nvim and $XDG_DATA_HOME/nvim/fff_queries, vicinae will share them so "
108+
"file-ranking learning carries over both ways. Disable if you hit fff schema/version errors.");
109+
reuseNvim.setDefaultValue(true);
110+
111+
return {indexing, paths, excludedPaths, watcherPaths, reuseNvim};
101112
}
102113

103114
void preferenceValuesChanged(const QJsonObject &preferences) const override {

0 commit comments

Comments
 (0)