feat: sync with upstream to 2.6

Author: konosubakonoakua

ChangeLog

@@ -1,6 +1,31 @@
 # SPDX-FileCopyrightText: Yorhel <[email protected]>
 # SPDX-License-Identifier: MIT
 
+2.6 - 2024-09-27
+	- Still requires Zig 0.12 or 0.13
+	- Add dependency on libzstd
+	- Add new export format to support threaded export and low-memory browsing
+	- Add `-O` and `--compress-level` CLI flags
+	- Add progress indicator to hardlink counting stage
+	- Fix displaying and exporting zero values when extended info is not available
+	- Fix clearing screen in some error cases
+	- Fix uncommon edge case in hardlink counting on refresh
+	- Use integer math instead of floating point to format numbers
+
+2.5 - 2024-07-24
+	- Still requires Zig 0.12 or 0.13
+	- Add parallel scanning with `-t,--threads` CLI flags
+	- Improve JSON export and import performance
+	- `--exclude-kernfs` is no longer checked on the top-level scan path
+	- Fix entries sometimes not showing up after refresh
+	- Fix file descriptor leak with `--exclude-caches` checking
+	- Fix possible crash on invalid UTF8 when scanning in `-1` UI mode
+	- Fix JSON export and import of the "other filesystem" flag
+	- Fix JSON import containing directories with a read error
+	- Fix mtime display of 'special' files
+	- Fix edge case bad performance when deleting hardlinks with many links
+	- Increased memory use for hardlinks (by ~10% in extreme cases, sorry)
+
 2.4 - 2024-04-21
 	- Now requires Zig 0.12
 	- Revert default color scheme back to 'off'

Makefile

@@ -9,7 +9,7 @@ ZIG ?= zig
 PREFIX ?= /usr/local
 BINDIR ?= ${PREFIX}/bin
 MANDIR ?= ${PREFIX}/share/man/man1
-ZIG_FLAGS ?= --release
+ZIG_FLAGS ?= --release=fast
 
 NCDU_VERSION=$(shell grep 'program_version = "' src/main.zig | sed -e 's/^.*"\(.\+\)".*$$/\1/')
 
@@ -52,9 +52,21 @@ dist:
 	rm -rf ncdu-${NCDU_VERSION}
 
 
-# ASSUMPTION: the ncurses source tree has been extracted into ncurses/
+# ASSUMPTION:
+# - the ncurses source tree has been extracted into ncurses/
+# - the zstd source tree has been extracted into zstd/
+# Would be nicer to do all this with the Zig build system, but no way am I
+# going to write build.zig's for these projects.
 static-%.tar.gz:
 	mkdir -p static-$*/nc static-$*/inst/pkg
+	cp -R zstd/lib static-$*/zstd
+	make -C static-$*/zstd -j8 libzstd.a V=1\
+		ZSTD_LIB_DICTBUILDER=0\
+		ZSTD_LIB_MINIFY=1\
+		ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1\
+		CC="${ZIG} cc --target=$*"\
+		LD="${ZIG} cc --target=$*"\
+		AR="${ZIG} ar" RANLIB="${ZIG} ranlib"
 	cd static-$*/nc && ../../ncurses/configure --prefix="`pwd`/../inst"\
 		--with-pkg-config-libdir="`pwd`/../inst/pkg"\
 		--without-cxx --without-cxx-binding --without-ada --without-manpages --without-progs\
@@ -71,8 +83,9 @@ static-%.tar.gz:
 	@#	--build-file ../build.zig --search-prefix inst/ --cache-dir zig -Drelease-fast=true
 	@# Alternative approach, bypassing zig-build
 	cd static-$* && ${ZIG} build-exe -target $*\
-		-Iinst/include -Iinst/include/ncursesw -lc inst/lib/libncursesw.a\
+		-Iinst/include -Iinst/include/ncursesw -Izstd -lc inst/lib/libncursesw.a zstd/libzstd.a\
 		--cache-dir zig-cache -static -fstrip -O ReleaseFast ../src/main.zig
+	strip -R .eh_frame -R .eh_frame_hdr static-$*/main
 	cd static-$* && mv main ncdu && tar -czf ../static-$*.tar.gz ncdu
 	rm -rf static-$*
 

README.md

@@ -20,9 +20,10 @@ C version (1.x).
 
 ## Requirements
 
-- Zig 0.12.0
+- Zig 0.12 or 0.13.
 - Some sort of POSIX-like OS
-- ncurses libraries and header files
+- ncurses
+- libzstd
 
 ## Install
 

build.zig

@@ -5,9 +5,7 @@ const std = @import("std");
 
 pub fn build(b: *std.Build) void {
     const target = b.standardTargetOptions(.{});
-    const optimize = b.standardOptimizeOption(.{
-        .preferred_optimize_mode = .ReleaseFast,
-    });
+    const optimize = b.standardOptimizeOption(.{});
 
     const pie = b.option(bool, "pie", "Build with PIE support (by default false)") orelse false;
 
@@ -21,6 +19,7 @@ pub fn build(b: *std.Build) void {
 
     exe.pie = pie;
     exe.root_module.linkSystemLibrary("ncursesw", .{});
+    exe.root_module.linkSystemLibrary("libzstd", .{});
     // https://github.com/ziglang/zig/blob/b52be973dfb7d1408218b8e75800a2da3dc69108/build.zig#L551-L554
     if (target.result.isDarwin()) {
         // useful for package maintainers

ncdu.1

@@ -1,15 +1,17 @@
 .\" SPDX-FileCopyrightText: Yorhel <[email protected]>
 .\" SPDX-License-Identifier: MIT
-.Dd April 20, 2024
+.Dd September 27, 2024
 .Dt NCDU 1
 .Os
 .Sh NAME
 .Nm ncdu
 .Nd NCurses Disk Usage
+.
 .Sh SYNOPSIS
 .Nm
 .Op Fl f Ar file
 .Op Fl o Ar file
+.Op Fl O Ar file
 .Op Fl e , \-extended , \-no\-extended
 .Op Fl \-ignore\-config
 .Op Fl x , \-one\-file\-system , \-cross\-file\-system
@@ -18,6 +20,8 @@
 .Op Fl \-include\-caches , \-exclude\-caches
 .Op Fl L , \-follow\-symlinks , \-no\-follow\-symlinks
 .Op Fl \-include\-kernfs , \-exclude\-kernfs
+.Op Fl t , \-threads Ar num
+.Op Fl \-compress\-level Ar num
 .Op Fl 0 , 1 , 2
 .Op Fl q , \-slow\-ui\-updates , \-fast\-ui\-updates
 .Op Fl \-enable\-shell , \-disable\-shell
@@ -44,11 +48,13 @@
 .Op Fl h , \-help
 .Nm
 .Op Fl v , V , \-version
+.
 .Sh DESCRIPTION
 .Nm
 (NCurses Disk Usage) is an interactive curses-based version of the well-known
 .Xr du 1 ,
 and provides a fast way to see what directories are using your disk space.
+.
 .Sh OPTIONS
 .Ss Mode Selection
 .Bl -tag -width Ds
@@ -59,10 +65,13 @@ Print version and quit.
 .It Fl f Ar file
 Load the given file, which has earlier been created with the
 .Fl o
+or
+.Fl O
 flag.
 If
 .Ar file
 is equivalent to '\-', the file is read from standard input.
+Reading from standard input is only supported for the JSON format.
 .Pp
 For the sake of preventing a screw-up, the current version of
 .Nm
@@ -73,7 +82,7 @@ will be disabled.
 .It Ar dir
 Scan the given directory.
 .It Fl o Ar file
-Export all necessary information to
+Export the directory tree in JSON format to
 .Ar file
 instead of opening the browser interface.
 If
@@ -87,6 +96,22 @@ directory with many files.
 uncompressed, or a little over 100 KiB when compressed with gzip.
 This scales linearly, so be prepared to handle a few tens of megabytes when
 dealing with millions of files.
+.Pp
+When running a multi-threaded scan or when scanning a directory tree that may
+not fit in memory, consider using
+.Fl O
+instead.
+.It Fl O Ar file
+Export the directory tree in binary format to
+.Ar file
+instead of opening the browser interface.
+If
+.Ar file
+is '\-', the data is written to standard output.
+The binary format has built-in compression, supports low-memory multi-threaded
+export (in combination with
+.Fl t )
+and can be browsed without importing the entire directory tree into memory.
 .It Fl e , \-extended , \-no\-extended
 Enable/disable extended information mode.
 This will, in addition to the usual file information, also read the ownership,
@@ -105,6 +130,7 @@ using 'm' and 'M', respectively.
 .It Fl \-ignore\-config
 Do not attempt to load any configuration files.
 .El
+.
 .Ss Scan Options
 These options affect the scanning progress, they have no effect when importing
 directory information from a file.
@@ -147,7 +173,28 @@ The exact counting behavior of this flag is subject to change in the future.
 .Pp
 The complete list of currently known pseudo filesystems is: binfmt, bpf, cgroup,
 cgroup2, debug, devpts, proc, pstore, security, selinux, sys, trace.
+.It Fl t , \-threads Ar num
+Number of threads to use when scanning the filesystem, defaults to 1.
+.Pp
+In single-threaded mode, the JSON export (see
+.Fl o )
+can operate with very little memory, but in multi-threaded mode the entire
+directory tree is first constructed in memory and written out after the
+filesystem scan has completed,
+This causes a delay in output and requires significantly more memory for large
+directory trees.
+The binary format (see
+.Fl O )
+does not have this problem and supports efficient exporting with any number of
+threads.
+.It Fl \-compress\-level Ar num
+Set the Zstandard compression level when using
+.Fl O
+to create a binary export.
+Valid values are 1 (fastest) to 19 (slowest).
+Defaults to 4.
 .El
+.
 .Ss Interface Options
 .Bl -tag -width Ds
 .It Fl 0
@@ -159,9 +206,8 @@ When exporting the data with
 ncurses will not be initialized at all.
 This option is the default when exporting to standard output.
 .It Fl 1
-Similar to
-.Fl 0 ,
-but does give feedback on the scanning progress with a single line of output.
+Write progress information to the terminal, but don't open a full-screen
+ncurses interface.
 This option is the default when exporting to a file.
 .Pp
 In some cases, the ncurses browser interface which you'll see after the
@@ -297,6 +343,7 @@ color scheme that also works in terminals with a light background.
 The default is
 .Ar off .
 .El
+.
 .Sh CONFIGURATION
 .Nm
 can be configured by placing command-line options in
@@ -325,6 +372,7 @@ Example configuration file:
 # Exclude .git directories
 \-\-exclude .git
 .Ed
+.
 .Sh KEYS
 .Bl -tag -width Ds
 .It ?
@@ -411,6 +459,7 @@ itself does not (currently) warn about or prevent this situation.
 .It q
 Quit
 .El
+.
 .Sh FILE FLAGS
 Entries in the browser interface may be prefixed by a one\-character flag.
 These flags have the following meaning:
@@ -434,6 +483,7 @@ Same file was already counted (hard link).
 .It e
 Empty directory.
 .El
+.
 .Sh EXAMPLES
 To scan and browse the directory you're currently in, all you need is a simple:
 .Dl ncdu
@@ -473,20 +523,23 @@ the local system without any network latency, and
 .Nm
 does not keep the entire directory structure in memory when exporting, so this
 won't consume much memory on the remote system.
+.
 .Sh SEE ALSO
 .Xr du 1 ,
 .Xr tree 1 .
 .Pp
 .Nm
 has a website:
 .Lk https://dev.yorhel.nl/ncdu
+.
 .Sh AUTHORS
 Written by
 .An Yorhel Aq Mt [email protected]
+.
 .Sh BUGS
 Directory hard links and firmlinks (MacOS) are not supported.
-They are not detected as being hard links, and will thus get scanned and
-counted multiple times.
+They are not detected as being hard links and will thus get scanned and counted
+multiple times.
 .Pp
 Some minor glitches may appear when displaying filenames that contain multibyte
 or multicolumn characters.