Added support for CLDF sources in stats cmd

Author: xrotwang

cldf/dataset.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"gocldf/internal/jsonutil"
+	"gocldf/internal/pathutil"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -75,6 +76,17 @@ func GetLoadedDataset(mdPath string, noChecks bool) (ds *Dataset, err error) {
 	return ds, nil
 }
 
+func (dataset *Dataset) TablePath(tbl *Table) (string, error) {
+	res := filepath.Join(filepath.Dir(dataset.MetadataPath), tbl.Url)
+	if !pathutil.PathExists(res) {
+		res += ".zip"
+	}
+	if !pathutil.PathExists(res) {
+		return "", errors.New("table path does not exist")
+	}
+	return res, nil
+}
+
 func (dataset *Dataset) LoadData(noChecks bool) error {
 	results := make(chan TableRead, len(dataset.Tables))
 	for _, tbl := range dataset.Tables {

cldf/sources.go

@@ -33,6 +33,7 @@ func NewSource(entry *bibtex.BibEntry) *Source {
 }
 
 type Sources struct {
+	Path       string
 	Items      []*Source
 	FieldNames []string
 }
@@ -61,7 +62,7 @@ func normalizeBibtex(r io.Reader) (io.Reader, error) {
 }
 
 func NewSources(p string) (sources *Sources, err error) {
-	f, err := pathutil.Reader(p)
+	pp, f, err := pathutil.Reader(p)
 	if err != nil {
 		return nil, err
 	}
@@ -90,8 +91,7 @@ func NewSources(p string) (sources *Sources, err error) {
 			}
 		}
 	}
-	return &Sources{Items: res, FieldNames: fields}, nil
-
+	return &Sources{Path: pp, Items: res, FieldNames: fields}, nil
 }
 
 func (s *Sources) SqlCreate() string {

cldf/table.go

@@ -159,7 +159,7 @@ func (tbl *Table) Read(dir string, dialect *Dialect, noChecks bool, ch chan<- Ta
 		rows [][]string
 		err  error
 	)
-	r, err := pathutil.Reader(fp)
+	_, r, err := pathutil.Reader(fp)
 	if err != nil {
 		ch <- TableRead{tbl.Url, err}
 		return

cmd/stats.go

@@ -4,7 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"gocldf/cldf"
+	"gocldf/internal/pathutil"
 	"io"
+	"path/filepath"
 	"text/tabwriter"
 
 	"github.com/spf13/cobra"
@@ -34,15 +36,42 @@ func stats(out io.Writer, mdPath string, withMetadata bool) error {
 	}
 	w := tabwriter.NewWriter(out, 0, 0, 1, ' ', tabwriter.Debug)
 	// noinspection GoUnhandledErrorResultInspection
-	fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "Filename", "Component", "Rows", "FKs")
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "Filename", "Component", "Rows", "Size")
 	// noinspection GoUnhandledErrorResultInspection
-	fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "--------", "---------", "----", "---")
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "--------", "---------", "--------", "----------")
 	for _, table := range ds.Tables {
+		path, err := ds.TablePath(table)
+		if err != nil {
+			return err
+		}
+		size, err := pathutil.GetFormattedSize(path)
+		if err != nil {
+			return err
+		}
 		cname := ""
 		if table.Comp != "" {
 			cname = table.CanonicalName
 		}
-		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", table.Url, cname, len(table.Data), len(table.ForeignKeys))
+		fmt.Fprintf(
+			w,
+			"%v\t%v\t%v\t%v\n",
+			filepath.Base(path),
+			cname,
+			fmt.Sprintf("%8s", fmt.Sprintf("%v", len(table.Data))),
+			fmt.Sprintf("%10s", size))
+	}
+	if ds.Sources != nil {
+		size, err := pathutil.GetFormattedSize(ds.Sources.Path)
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(
+			w,
+			"%v\t%v\t%v\t%v\n",
+			filepath.Base(ds.Sources.Path),
+			"SourceTable",
+			fmt.Sprintf("%8s", fmt.Sprintf("%v", len(ds.Sources.Items))),
+			fmt.Sprintf("%10s", size))
 	}
 	// noinspection GoUnhandledErrorResultInspection
 	w.Flush()

internal/pathutil/path.go

@@ -4,10 +4,29 @@ import (
 	"archive/zip"
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 )
 
+func GetFormattedSize(path string) (string, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return "", err
+	}
+	// Size() returns the size in bytes as an int64
+	units := []string{"bytes", "KB", "MB", "GB"}
+	size := float64(info.Size())
+
+	for _, unit := range units {
+		if size < 1024.0 && size > -1024.0 {
+			return fmt.Sprintf("%.1f%v", size, unit), nil
+		}
+		size = size / 1024.0
+	}
+	return fmt.Sprintf("%g%v", size, "TB"), nil
+}
+
 func PathExists(path string) bool {
 	_, err := os.Stat(path)
 	if err == nil {
@@ -75,17 +94,17 @@ Usage:
 		}
 	}(reader)
 */
-func Reader(p string) (r any, err error) {
+func Reader(p string) (pp string, r any, err error) {
 	if !PathExists(p) {
 		zippedBytes, err := readZipped(p + ".zip")
 		if err != nil {
-			return nil, err
+			return "", nil, err
 		}
-		return bytes.NewReader(zippedBytes), nil
+		return p + ".zip", bytes.NewReader(zippedBytes), nil
 	}
 	file, err := os.Open(p)
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
-	return file, nil
+	return p, file, nil
 }

internal/pathutil/path_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func read(fname string) (string, error) {
-	r, err := Reader(filepath.Join("testdata", fname))
+	_, r, err := Reader(filepath.Join("testdata", fname))
 	if err != nil {
 		return "", err
 	}

test/test_regression.py

@@ -7,6 +7,8 @@
 import pathlib
 import subprocess
 
+from clldutils.misc import format_size
+
 PROJECTS = pathlib.Path(os.path.expanduser('~')) / "projects"
 DATASETS = [
     "grambank/grambank-cldf/cldf/StructureDataset-metadata.json",
@@ -28,11 +30,12 @@ def run():
         shutil.copy(wd / "gocldf", bin)
         assert bin.exists()
         for ds in DATASETS:
+            out = temp / "db.sqlite"
             s = time.time()
             print("{} ...".format(ds))
-            res = subprocess.check_output([str(bin), "createdb", str(PROJECTS / ds), "db.sqlite", "-f"])
+            res = subprocess.check_output([str(bin), "createdb", str(PROJECTS / ds), str(out), "-f"])
             assert "Loaded" in res.decode("utf8")
-            print("... {:.1f}s".format(time.time()-s))
+            print("... {:.1f}s\t{}".format(time.time()-s, format_size(out.stat().st_size)))
 
 if __name__ == "__main__":
     run()