Added support for CLDF sources

Author: xrotwang

RELEASING.md

@@ -4,5 +4,9 @@ go mod tidy
 
 go test ./...
 
+~/venvs/cldf/bin/python test/test_regression.py
+
+FIXME: get cross-compilation set up for goreleaser (e.g. )
+
 goreleaser check
 goreleaser release --snapshot --clean

cldf/dataset.go

@@ -1,10 +1,9 @@
 package cldf
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
+	"gocldf/internal/jsonutil"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -19,18 +18,11 @@ type Dataset struct {
 }
 
 func NewDataset(mdPath string) (*Dataset, error) {
-	data, err := os.ReadFile(mdPath)
+	result, err := jsonutil.ReadObject(mdPath)
 	if err != nil {
 		return nil, err
 	}
-	var result map[string]interface{}
-
-	err = json.Unmarshal(data, &result)
-	if err != nil {
-		return nil, err
-	}
-
-	metadata := make(map[string]interface{}, len(result)-1)
+	metadata := make(map[string]any, len(result)-1)
 	for k, v := range result {
 		if k == "tables" {
 			continue
@@ -41,12 +33,11 @@ func NewDataset(mdPath string) (*Dataset, error) {
 		sourcesBibtex string
 		sources       *Sources
 	)
-	val, ok := result["dc:source"]
-	if ok {
-		sourcesBibtex, ok = val.(string)
-		if !ok {
-			return nil, errors.New("invalid dc:source")
-		}
+	sourcesBibtex, err = jsonutil.GetString(result, "dc:source", "")
+	if err != nil {
+		return nil, err
+	}
+	if sourcesBibtex != "" {
 		sources, err = NewSources(filepath.Join(filepath.Dir(mdPath), sourcesBibtex))
 		if err != nil {
 			return nil, err
@@ -115,10 +106,10 @@ func (dataset *Dataset) UrlToCanonicalName() map[string]string {
 	return res
 }
 
+// orderedTables determines the order in which to create the tables in a db in such a way that foreign key constraints are satisfied.
 func (dataset *Dataset) orderedTables() ([]*Table, error) {
 	var (
-		urlToName = dataset.UrlToCanonicalName()
-		// Determine the order in which to create the tables
+		urlToName     = dataset.UrlToCanonicalName()
 		tables        []string
 		orderedTables []string
 	)
@@ -207,11 +198,9 @@ type TableData struct {
 	Rows      [][]any
 }
 
-// Function ToSqlite returns the data necessary to load the dataset into a SQLite database.
-func (dataset *Dataset) ToSqlite(noChecks bool) (string, []TableData, error) {
-	var tableData []TableData
-
-	schema, err := dataset.sqlSchema(noChecks)
+// ToSqlite returns the data necessary to load the dataset into a SQLite database.
+func (dataset *Dataset) ToSqlite(noChecks bool) (schema string, tableData []TableData, err error) {
+	schema, err = dataset.sqlSchema(noChecks)
 	if err != nil {
 		return "", tableData, err
 	}

cldf/sources.go

@@ -1,8 +1,12 @@
 package cldf
 
 import (
+	"bufio"
 	"fmt"
+	"gocldf/internal/pathutil"
+	"io"
 	"os"
+	"regexp"
 	"slices"
 	"strings"
 
@@ -18,7 +22,8 @@ type Source struct {
 func NewSource(entry *bibtex.BibEntry) *Source {
 	fields := make(map[string]string)
 	for k, v := range entry.Fields {
-		fields[k] = v.String()
+		// We reverse the temporary replacement for @ to appease the BibTeX parser.
+		fields[k] = strings.ReplaceAll(v.String(), "�", "@")
 	}
 	return &Source{
 		Id:     entry.CiteName,
@@ -32,20 +37,54 @@ type Sources struct {
 	FieldNames []string
 }
 
-func NewSources(p string) (*Sources, error) {
-	f, err := os.Open(p)
+func normalizeBibtex(r io.Reader) (io.Reader, error) {
+	var res []string
+	comment := regexp.MustCompile("^\\s*comment\\s*=")
+	atAtStart := regexp.MustCompile("^\\s*@")
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if comment.MatchString(line) {
+			// For some reason "comment" seems to be forbidden as field name.
+			line = strings.Replace(line, "comment", "comments", 1)
+		}
+		if !atAtStart.MatchString(line) {
+			line = strings.ReplaceAll(line, "@", "�")
+		}
+		res = append(res, line)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return strings.NewReader(strings.Join(res, "\n")), nil
+}
+
+func NewSources(p string) (sources *Sources, err error) {
+	f, err := pathutil.Reader(p)
 	if err != nil {
 		return nil, err
 	}
-	entries, err := bibtex.Parse(f)
+	defer func(file any) {
+		switch file.(type) {
+		case *os.File:
+			err = file.(*os.File).Close()
+		}
+	}(f)
+
+	r, err := normalizeBibtex(f.(io.Reader))
 	if err != nil {
 		return nil, err
 	}
+	entries, err := bibtex.Parse(r)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %v: %w", p, err)
+	}
 	res := make([]*Source, len(entries.Entries))
-	fields := []string{}
+	var fields []string
 	for i, entry := range entries.Entries {
 		res[i] = NewSource(entry)
-		for name, _ := range entry.Fields {
+		for name := range entry.Fields {
 			if !slices.Contains(fields, name) {
 				fields = append(fields, name)
 			}
@@ -56,11 +95,14 @@ func NewSources(p string) (*Sources, error) {
 }
 
 func (s *Sources) SqlCreate() string {
-	res := []string{}
+	var res []string
 	res = append(res, "CREATE TABLE IF NOT EXISTS `SourceTable` (")
 	res = append(res, "\t`id`\tTEXT,")
 	res = append(res, "\t`genre`\tTEXT,")
 	for _, field := range s.FieldNames {
+		if field == "type" || field == "id" {
+			field += "_"
+		}
 		res = append(res, fmt.Sprintf("\t`%s`\tTEXT,", field))
 	}
 	res = append(res, "\tPRIMARY KEY(`id`)")
@@ -80,6 +122,9 @@ func (s *Sources) itemsToSql() (rows [][]any, colNames []string, err error) {
 		rows[i][1] = item.Type
 
 		for j, field := range s.FieldNames {
+			if field == "type" || field == "id" {
+				field += "_"
+			}
 			if i == 0 {
 				colNames = append(colNames, field)
 			}

cldf/table.go

@@ -1,8 +1,6 @@
 package cldf
 
 import (
-	"archive/zip"
-	"bytes"
 	"encoding/csv"
 	"encoding/json"
 	"errors"
@@ -41,8 +39,6 @@ type Table struct {
 }
 
 func NewTable(jsonTable map[string]interface{}, withSourceTable bool) (tbl *Table, err error) {
-	// FIXME: We must know whether the dataset has a sources bibfile, because then we'll turn
-	// columns with canonical name cldf_source into list-valued foreign keys to SourceTable.
 	var (
 		dialect *Dialect
 		trimmer = func(s string) string { return s }
@@ -57,7 +53,7 @@ func NewTable(jsonTable map[string]interface{}, withSourceTable bool) (tbl *Tabl
 		if err != nil {
 			return nil, err
 		}
-		if col.CanonicalName == "cldf_source" {
+		if withSourceTable && col.CanonicalName == "cldf_source" {
 			// remember and store additional foreign key constraint!
 			fks = append(
 				fks,
@@ -156,68 +152,26 @@ type TableRead struct {
 	Err error
 }
 
-func readZipped(fp string) (bytes []byte, err error) {
-	r, err := zip.OpenReader(fp)
-	if err != nil {
-		return nil, err
-	}
-	defer func(r *zip.ReadCloser) {
-		err = r.Close()
-	}(r)
-
-	var contentBytes []byte
-	for _, f := range r.File {
-		rc, err := f.Open()
-		if err != nil {
-			return nil, err
-		}
-		contentBytes, err = io.ReadAll(rc)
-		if err != nil {
-			return nil, err
-		}
-		err = rc.Close()
-		if err != nil {
-			return nil, err
-		} // Must close each file reader individually
-		break
-	}
-	return contentBytes, nil
-}
-
 func (tbl *Table) Read(dir string, dialect *Dialect, noChecks bool, ch chan<- TableRead) {
 	var reader *csv.Reader
 	fp := filepath.Join(dir, tbl.Url)
-	zipped := false
-	if !pathutil.PathExists(fp) {
-		fp += ".zip"
-		zipped = true
-	}
 	var (
 		rows [][]string
 		err  error
 	)
-	if zipped {
-		zippedBytes, err := readZipped(fp)
-		if err != nil {
-			ch <- TableRead{tbl.Url, err}
-			return
-		}
-		reader = csv.NewReader(bytes.NewReader(zippedBytes))
-	} else {
-		file, err := os.Open(fp)
-		if err != nil {
-			ch <- TableRead{tbl.Url, err}
-			return
-		}
-		defer func(file *os.File) {
-			err := file.Close()
-			if err != nil {
-				ch <- TableRead{tbl.Url, err}
-				return
-			}
-		}(file)
-		reader = csv.NewReader(file)
+	r, err := pathutil.Reader(fp)
+	if err != nil {
+		ch <- TableRead{tbl.Url, err}
+		return
 	}
+	defer func(file any) {
+		switch file.(type) {
+		case *os.File:
+			err = file.(*os.File).Close()
+		}
+	}(r)
+	reader = csv.NewReader(r.(io.Reader))
+
 	if tbl.Dialect != nil {
 		dialect = tbl.Dialect
 	}
@@ -296,9 +250,8 @@ func (tbl *Table) associationTableRowsToSql(
 		tpk     string
 		colName string
 	)
-	nameToCol := tbl.nameToCol()
 	stable := tbl.CanonicalName
-	spk := nameToCol[tbl.PrimaryKey[0]].CanonicalName
+	spk := tbl.nameToCol()[tbl.PrimaryKey[0]].CanonicalName
 
 	if fk.Reference.Resource == "SourceTable" {
 		ttable = "SourceTable"
@@ -310,8 +263,8 @@ func (tbl *Table) associationTableRowsToSql(
 			panic("not found " + fk.Reference.Resource)
 		}
 		ttable = ttable_.CanonicalName
-		tpk = nameToCol[ttable_.PrimaryKey[0]].CanonicalName
-		colName = nameToCol[fk.ColumnReference[0]].CanonicalName
+		tpk = ttable_.nameToCol()[ttable_.PrimaryKey[0]].CanonicalName
+		colName = tbl.nameToCol()[fk.ColumnReference[0]].CanonicalName
 	}
 
 	colNames = []string{

cmd/createdb.go

@@ -2,28 +2,20 @@ package cmd
 
 import (
 	"database/sql"
-	"errors"
 	"fmt"
 	"gocldf/cldf"
 	"gocldf/internal/dbutil"
 	"gocldf/internal/pathutil"
 	"io"
-	"os"
 	"slices"
 
 	"github.com/spf13/cobra"
 )
 
 func createdb(out io.Writer, mdPath string, dbPath string, overwrite bool, noChecks bool) (err error) {
-	if pathutil.PathExists(dbPath) {
-		if overwrite {
-			err = os.Remove(dbPath)
-			if err != nil {
-				return err
-			}
-		} else {
-			return errors.New("database already exists")
-		}
+	dbPath, err = pathutil.GetFreshPath(dbPath, overwrite)
+	if err != nil {
+		return err
 	}
 	ds, err := cldf.GetLoadedDataset(mdPath, noChecks)
 	if err != nil {
@@ -40,7 +32,10 @@ func createdb(out io.Writer, mdPath string, dbPath string, overwrite bool, noChe
 				return err
 			}
 			for _, tData := range tableData { // ... and the data.
-				dbutil.BatchInsert(tx, tData.TableName, tData.ColNames, tData.Rows)
+				err = dbutil.BatchInsert(tx, tData.TableName, tData.ColNames, tData.Rows)
+				if err != nil {
+					return err
+				}
 			}
 			return nil
 		})