feat: sync with upstream to 2.7

Author: konosubakonoakua

.gitignore

@@ -1,2 +1,3 @@
 zig-out/
+.zig-cache/
 zig-cache/

ChangeLog

@@ -1,6 +1,15 @@
 # SPDX-FileCopyrightText: Yorhel <[email protected]>
 # SPDX-License-Identifier: MIT
 
+2.7 - 2024-11-19
+	- Still requires Zig 0.12 or 0.13
+	- Support transparent reading/writing of zstandard-compressed JSON
+	- Add `--compress` and `--export-block-size` options
+	- Perform tilde expansion on paths in the config file
+	- Fix JSON import of escaped UTF-16 surrogate pairs
+	- Fix incorrect field in root item when exporting to the binary format
+	- Add -Dstrip build flag
+
 2.6 - 2024-09-27
 	- Still requires Zig 0.12 or 0.13
 	- Add dependency on libzstd

Makefile

@@ -9,7 +9,7 @@ ZIG ?= zig
 PREFIX ?= /usr/local
 BINDIR ?= ${PREFIX}/bin
 MANDIR ?= ${PREFIX}/share/man/man1
-ZIG_FLAGS ?= --release=fast
+ZIG_FLAGS ?= --release=fast -Dstrip
 
 NCDU_VERSION=$(shell grep 'program_version = "' src/main.zig | sed -e 's/^.*"\(.\+\)".*$$/\1/')
 
@@ -68,24 +68,25 @@ static-%.tar.gz:
 		LD="${ZIG} cc --target=$*"\
 		AR="${ZIG} ar" RANLIB="${ZIG} ranlib"
 	cd static-$*/nc && ../../ncurses/configure --prefix="`pwd`/../inst"\
-		--with-pkg-config-libdir="`pwd`/../inst/pkg"\
 		--without-cxx --without-cxx-binding --without-ada --without-manpages --without-progs\
-		--without-tests --enable-pc-files --without-pkg-config --without-shared --without-debug\
+		--without-tests --disable-pc-files --without-pkg-config --without-shared --without-debug\
 		--without-gpm --without-sysmouse --enable-widec --with-default-terminfo-dir=/usr/share/terminfo\
 		--with-terminfo-dirs=/usr/share/terminfo:/lib/terminfo:/usr/local/share/terminfo\
 		--with-fallbacks="screen linux vt100 xterm xterm-256color" --host=$*\
 		CC="${ZIG} cc --target=$*"\
 		LD="${ZIG} cc --target=$*"\
 		AR="${ZIG} ar" RANLIB="${ZIG} ranlib"\
-		CPPFLAGS=-D_GNU_SOURCE && make -j8 && make install.libs
+		CPPFLAGS=-D_GNU_SOURCE && make -j8
 	@# zig-build - cleaner approach but doesn't work, results in a dynamically linked binary.
 	@#cd static-$* && PKG_CONFIG_LIBDIR="`pwd`/inst/pkg" zig build -Dtarget=$*
 	@#	--build-file ../build.zig --search-prefix inst/ --cache-dir zig -Drelease-fast=true
 	@# Alternative approach, bypassing zig-build
 	cd static-$* && ${ZIG} build-exe -target $*\
-		-Iinst/include -Iinst/include/ncursesw -Izstd -lc inst/lib/libncursesw.a zstd/libzstd.a\
+		-Inc/include -Izstd -lc nc/lib/libncursesw.a zstd/libzstd.a\
 		--cache-dir zig-cache -static -fstrip -O ReleaseFast ../src/main.zig
-	strip -R .eh_frame -R .eh_frame_hdr static-$*/main
+	@# My system's strip can't deal with arm binaries and zig doesn't wrap a strip alternative.
+	@# Whatever, just let it error for those.
+	strip -R .eh_frame -R .eh_frame_hdr static-$*/main || true
 	cd static-$* && mv main ncdu && tar -czf ../static-$*.tar.gz ncdu
 	rm -rf static-$*
 

README.md

@@ -6,7 +6,7 @@ SPDX-License-Identifier: MIT
 # ncdu-zig
 
 ## Description
-<img src="https://github.com/user-attachments/assets/f2b4b591-ac61-4751-8d59-fbaf4c747521" width="50%" />
+<img src="https://github.com/user-attachments/assets/a0235a3e-5071-406b-83ce-7343ab0268e1" width="50%" />
 
 Ncdu is a disk usage analyzer with an ncurses interface. It is designed to find
 space hogs on a remote server where you don't have an entire graphical setup

build.zig

@@ -8,12 +8,14 @@ pub fn build(b: *std.Build) void {
     const optimize = b.standardOptimizeOption(.{});
 
     const pie = b.option(bool, "pie", "Build with PIE support (by default false)") orelse false;
+    const strip = b.option(bool, "strip", "Strip debugging info (by default false)") orelse false;
 
     const exe = b.addExecutable(.{
         .name = "ncdu",
         .root_source_file = b.path("src/main.zig"),
         .target = target,
         .optimize = optimize,
+        .strip = strip,
         .link_libc = true,
     });
 
@@ -44,6 +46,7 @@ pub fn build(b: *std.Build) void {
     });
     unit_tests.pie = pie;
     unit_tests.root_module.linkSystemLibrary("ncursesw", .{});
+    unit_tests.root_module.linkSystemLibrary("libzstd", .{});
 
     const run_unit_tests = b.addRunArtifact(unit_tests);
 

ncdu.1

@@ -1,6 +1,6 @@
 .\" SPDX-FileCopyrightText: Yorhel <[email protected]>
 .\" SPDX-License-Identifier: MIT
-.Dd September 27, 2024
+.Dd November 19, 2024
 .Dt NCDU 1
 .Os
 .Sh NAME
@@ -21,7 +21,9 @@
 .Op Fl L , \-follow\-symlinks , \-no\-follow\-symlinks
 .Op Fl \-include\-kernfs , \-exclude\-kernfs
 .Op Fl t , \-threads Ar num
+.Op Fl c , \-compress , \-no\-compress
 .Op Fl \-compress\-level Ar num
+.Op Fl \-export\-block\-size Ar num
 .Op Fl 0 , 1 , 2
 .Op Fl q , \-slow\-ui\-updates , \-fast\-ui\-updates
 .Op Fl \-enable\-shell , \-disable\-shell
@@ -97,6 +99,11 @@ uncompressed, or a little over 100 KiB when compressed with gzip.
 This scales linearly, so be prepared to handle a few tens of megabytes when
 dealing with millions of files.
 .Pp
+Consider enabling
+.Fl c
+to output Zstandard-compressed JSON, which can significantly reduce size of the
+exported data.
+.Pp
 When running a multi-threaded scan or when scanning a directory tree that may
 not fit in memory, consider using
 .Fl O
@@ -187,12 +194,36 @@ The binary format (see
 .Fl O )
 does not have this problem and supports efficient exporting with any number of
 threads.
+.El
+.
+.Ss Export Options
+These options affect behavior when exporting to file with the
+.Fl o
+or
+.Fl O
+options.
+.Bl -tag -width Ds
+.It Fl c , \-compress , \-no\-compress
+Enable or disable Zstandard compression when exporting to JSON (see
+.Fl o ) .
 .It Fl \-compress\-level Ar num
 Set the Zstandard compression level when using
 .Fl O
-to create a binary export.
+or
+.Fl c .
 Valid values are 1 (fastest) to 19 (slowest).
 Defaults to 4.
+.It Fl \-export\-block\-size Ar num
+Set the block size, in kibibytes, for the binary export format (see
+.Fl O ) .
+Larger blocks require more memory but result in better compression efficiency.
+This option can be combined with a higher
+.Fl \-compress\-level
+for even better compression.
+.Pp
+Accepted values are between 4 and 16000.
+The defaults is to start at 64 KiB and then gradually increase the block size
+for large exports.
 .El
 .
 .Ss Interface Options
@@ -487,34 +518,28 @@ Empty directory.
 .Sh EXAMPLES
 To scan and browse the directory you're currently in, all you need is a simple:
 .Dl ncdu
-If you want to scan a full filesystem, for example your root filesystem, then
-you'll want to use
+To scan a full filesystem, for example your root filesystem, you'll want to use
 .Fl x :
 .Dl ncdu \-x /
 .Pp
 Since scanning a large directory may take a while, you can scan a directory and
 export the results for later viewing:
 .Bd -literal -offset indent
-ncdu \-1xo\- / | gzip >export.gz
+ncdu \-1xO export.ncdu /
 # ...some time later:
-zcat export.gz | ncdu \-f\-
+ncdu \-f export.ncdu
 .Ed
 To export from a cron job, make sure to replace
 .Fl 1
 with
 .Fl 0
-to suppress any unnecessary output.
+to suppress unnecessary progress output.
 .Pp
 You can also export a directory and browse it once scanning is done:
-.Dl ncdu \-o\- | tee export.file | ./ncdu \-f\-
-The same is possible with gzip compression, but is a bit kludgey:
-.Dl ncdu \-o\- | gzip | tee export.gz | gunzip | ./ncdu \-f\-
+.Dl ncdu \-co\- | tee export.json.zst | ./ncdu \-f\-
 .Pp
 To scan a system remotely, but browse through the files locally:
-.Dl ssh \-C user@system ncdu \-o\- / | ./ncdu \-f\-
-The
-.Fl C
-option to ssh enables compression, which will be very useful over slow links.
+.Dl ssh user@system ncdu \-co\- / | ./ncdu \-f\-
 Remote scanning and local viewing has two major advantages when
 compared to running
 .Nm

src/bin_export.zig

@@ -7,9 +7,7 @@ const model = @import("model.zig");
 const sink = @import("sink.zig");
 const util = @import("util.zig");
 const ui = @import("ui.zig");
-
-extern fn ZSTD_compress(dst: ?*anyopaque, dstCapacity: usize, src: ?*const anyopaque, srcSize: usize, compressionLevel: c_int) usize;
-extern fn ZSTD_isError(code: usize) c_uint;
+const c = @import("c.zig").c;
 
 pub const global = struct {
     var fd: std.fs.File = undefined;
@@ -65,9 +63,6 @@ inline fn blockHeader(id: u4, len: u28) [4]u8 { return bigu32((@as(u32, id) << 2
 
 inline fn cborByte(major: CborMajor, arg: u5) u8 { return (@as(u8, @intFromEnum(major)) << 5) | arg; }
 
-// ZSTD_COMPRESSBOUND(), assuming input does not exceed ZSTD_MAX_INPUT_SIZE
-fn compressBound(size: usize) usize { return size + (size>>8) + (if (size < (128<<10)) ((128<<10) - size) >> 11 else 0); }
-
 
 // (Uncompressed) data block size.
 // Start with 64k, then use increasingly larger block sizes as the export file
@@ -77,7 +72,8 @@ fn compressBound(size: usize) usize { return size + (size>>8) + (if (size < (128
 fn blockSize(num: u32) usize {
     //                        block size    uncompressed data in this num range
     //                 # mil      # KiB         # GiB
-    return if (num < ( 1<<20))   64<<10  //    64
+    return main.config.export_block_size
+    orelse if (num < ( 1<<20))   64<<10  //    64
       else if (num < ( 2<<20))  128<<10  //   128
       else if (num < ( 4<<20))  256<<10  //   512
       else if (num < ( 8<<20))  512<<10  //  2048
@@ -100,8 +96,8 @@ pub const Thread = struct {
 
     fn compressZstd(in: []const u8, out: []u8) usize {
         while (true) {
-            const r = ZSTD_compress(out.ptr, out.len, in.ptr, in.len, main.config.complevel);
-            if (ZSTD_isError(r) == 0) return r;
+            const r = c.ZSTD_compress(out.ptr, out.len, in.ptr, in.len, main.config.complevel);
+            if (c.ZSTD_isError(r) == 0) return r;
             ui.oom(); // That *ought* to be the only reason the above call can fail.
         }
     }
@@ -110,7 +106,7 @@ pub const Thread = struct {
         var out = std.ArrayList(u8).init(main.allocator);
         if (t.block_num == std.math.maxInt(u32) or t.off == 0) return out;
 
-        out.ensureTotalCapacityPrecise(12 + compressBound(t.off)) catch unreachable;
+        out.ensureTotalCapacityPrecise(12 + @as(usize, @intCast(c.ZSTD_COMPRESSBOUND(@as(c_int, @intCast(t.off)))))) catch unreachable;
         out.items.len = out.capacity;
         const bodylen = compressZstd(t.buf[0..t.off], out.items[8..]);
         out.items.len = 12 + bodylen;
@@ -395,7 +391,7 @@ pub const Dir = struct {
             p.last_sub = t.itemStart(.dir, p.last_sub, name);
         } else {
             d.countLinks(null);
-            global.root_itemref = t.itemStart(.dir, 0, name);
+            global.root_itemref = t.itemStart(.dir, null, name);
         }
         d.inodes.deinit();
 

src/bin_reader.zig

@@ -8,9 +8,7 @@ const util = @import("util.zig");
 const sink = @import("sink.zig");
 const ui = @import("ui.zig");
 const bin_export = @import("bin_export.zig");
-
-extern fn ZSTD_decompress(dst: ?*anyopaque, dstCapacity: usize, src: ?*const anyopaque, compressedSize: usize) usize;
-extern fn ZSTD_getFrameContentSize(src: ?*const anyopaque, srcSize: usize) c_ulonglong;
+const c = @import("c.zig").c;
 
 
 const CborMajor = bin_export.CborMajor;
@@ -103,11 +101,11 @@ fn readBlock(num: u32) []const u8 {
         catch |e| ui.die("Error reading from file: {s}\n", .{ui.errorString(e)});
     if (rdlen != buf.len) die();
 
-    const rawlen = ZSTD_getFrameContentSize(buf.ptr, buf.len);
+    const rawlen = c.ZSTD_getFrameContentSize(buf.ptr, buf.len);
     if (rawlen <= 0 or rawlen >= (1<<24)) die();
     block.data = main.allocator.alloc(u8, @intCast(rawlen)) catch unreachable;
 
-    const res = ZSTD_decompress(block.data.ptr, block.data.len, buf.ptr, buf.len);
+    const res = c.ZSTD_decompress(block.data.ptr, block.data.len, buf.ptr, buf.len);
     if (res != block.data.len) ui.die("Error decompressing block {} (expected {} got {})\n", .{ num, block.data.len, res });
 
     return block.data;

src/browser.zig

@@ -9,7 +9,7 @@ const mem_sink = @import("mem_sink.zig");
 const bin_reader = @import("bin_reader.zig");
 const delete = @import("delete.zig");
 const ui = @import("ui.zig");
-const c = @cImport(@cInclude("time.h"));
+const c = @import("c.zig").c;
 const util = @import("util.zig");
 
 // Currently opened directory.
@@ -644,8 +644,8 @@ const info = struct {
     fn keyInput(ch: i32) bool {
         if (entry.?.pack.etype == .link) {
             switch (ch) {
-                '1', 'h', ui.c.KEY_LEFT => { set(entry, .info); return true; },
-                '2', 'l', ui.c.KEY_RIGHT => { set(entry, .links); return true; },
+                '1', 'h', c.KEY_LEFT => { set(entry, .info); return true; },
+                '2', 'l', c.KEY_RIGHT => { set(entry, .links); return true; },
                 else => {},
             }
         }
@@ -802,17 +802,17 @@ const help = struct {
             '1' => tab = .keys,
             '2' => tab = .flags,
             '3' => tab = .about,
-            'h', ui.c.KEY_LEFT => tab = if (tab == .about) .flags else .keys,
-            'l', ui.c.KEY_RIGHT => tab = if (tab == .keys) .flags else .about,
-            'j', ' ', ui.c.KEY_DOWN, ui.c.KEY_NPAGE => {
+            'h', c.KEY_LEFT => tab = if (tab == .about) .flags else .keys,
+            'l', c.KEY_RIGHT => tab = if (tab == .keys) .flags else .about,
+            'j', ' ', c.KEY_DOWN, c.KEY_NPAGE => {
                 const max = switch (tab) {
                     .keys => keys.len/2 - keylines,
                     else => @as(u32, 0),
                 };
                 if (offset < max)
                     offset += 1;
             },
-            'k', ui.c.KEY_UP, ui.c.KEY_PPAGE => { if (offset > 0) offset -= 1; },
+            'k', c.KEY_UP, c.KEY_PPAGE => { if (offset > 0) offset -= 1; },
             else => state = .main,
         }
     }
@@ -917,16 +917,16 @@ fn sortToggle(col: main.config.SortCol, default_order: main.config.SortOrder) vo
 
 fn keyInputSelection(ch: i32, idx: *usize, len: usize, page: u32) bool {
     switch (ch) {
-        'j', ui.c.KEY_DOWN => {
+        'j', c.KEY_DOWN => {
             if (idx.*+1 < len) idx.* += 1;
         },
-        'k', ui.c.KEY_UP => {
+        'k', c.KEY_UP => {
             if (idx.* > 0) idx.* -= 1;
         },
-        ui.c.KEY_HOME => idx.* = 0,
-        ui.c.KEY_END, ui.c.KEY_LL => idx.* = len -| 1,
-        ui.c.KEY_PPAGE => idx.* = idx.* -| page,
-        ui.c.KEY_NPAGE => idx.* = @min(len -| 1, idx.* + page),
+        c.KEY_HOME => idx.* = 0,
+        c.KEY_END, c.KEY_LL => idx.* = len -| 1,
+        c.KEY_PPAGE => idx.* = idx.* -| page,
+        c.KEY_NPAGE => idx.* = @min(len -| 1, idx.* + page),
         else => return false,
     }
     return true;
@@ -1017,7 +1017,7 @@ pub fn keyInput(ch: i32) void {
         },
 
         // Navigation
-        10, 'l', ui.c.KEY_RIGHT => {
+        10, 'l', c.KEY_RIGHT => {
             if (dir_items.items.len == 0) {
             } else if (dir_items.items[cursor_idx]) |e| {
                 if (e.dir()) |d| {
@@ -1032,7 +1032,7 @@ pub fn keyInput(ch: i32) void {
                 state = .main;
             }
         },
-        'h', '<', ui.c.KEY_BACKSPACE, ui.c.KEY_LEFT => {
+        'h', '<', c.KEY_BACKSPACE, c.KEY_LEFT => {
             if (dir_parents.items.len > 1) {
                 //const h = dir_parent.entry.nameHash();
                 enterParent();

src/c.zig

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: Yorhel <[email protected]>
+// SPDX-License-Identifier: MIT
+
+pub const c = @cImport({
+    @cDefine("_XOPEN_SOURCE", "1"); // for wcwidth()
+    @cInclude("stdio.h"); // fopen(), used to initialize ncurses
+    @cInclude("string.h"); // strerror()
+    @cInclude("time.h"); // strftime()
+    @cInclude("wchar.h"); // wcwidth()
+    @cInclude("locale.h"); // setlocale() and localeconv()
+    @cInclude("fnmatch.h"); // fnmatch()
+    @cInclude("unistd.h"); // getuid()
+    @cInclude("sys/types.h"); // struct passwd
+    @cInclude("pwd.h"); // getpwnam(), getpwuid()
+    if (@import("builtin").os.tag == .linux) {
+        @cInclude("sys/vfs.h"); // statfs()
+    }
+    @cInclude("curses.h");
+    @cInclude("zstd.h");
+});