witch to idiomatic error handling

Author: xrotwang

csvw/column/column.go

@@ -16,7 +16,7 @@ type Column struct {
 	Null          []string
 }
 
-func New(index int, jsonCol map[string]interface{}) *Column {
+func New(index int, jsonCol map[string]interface{}) (*Column, error) {
 	var (
 		name          string = ""
 		purl          string = ""
@@ -56,13 +56,18 @@ func New(index int, jsonCol map[string]interface{}) *Column {
 	} else {
 		canonicalName = name
 	}
-	return &Column{
+	dt, err := datatype.New(jsonCol)
+	if err != nil {
+		return nil, err
+	}
+	col := &Column{
 		Name:          name,
 		CanonicalName: canonicalName,
 		PropertyUrl:   purl,
-		Datatype:      *datatype.New(jsonCol),
+		Datatype:      *dt,
 		Separator:     sep,
 		Null:          null}
+	return col, nil
 }
 
 func (column *Column) ToGo(s string, split bool) (any, error) {
@@ -87,6 +92,10 @@ func (column *Column) ToGo(s string, split bool) (any, error) {
 	return column.Datatype.ToGo(s)
 }
 
+func (column *Column) ToSql(x any) (any, error) {
+	return column.Datatype.ToSql(x)
+}
+
 func (column *Column) ToString(x any) (string, error) {
 	if x == nil {
 		return column.Null[0], nil

csvw/column/column_test.go

@@ -12,7 +12,11 @@ func makeCol(jsonString string) Column {
 	if err != nil {
 		panic(err)
 	}
-	return *New(0, result)
+	col, err := New(0, result)
+	if err != nil {
+		panic(err)
+	}
+	return *col
 }
 
 func TestColumn_CanonicalName(t *testing.T) {

csvw/dataset/dataset.go

@@ -3,14 +3,13 @@ package dataset
 import (
 	"database/sql"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"gocldf/csvw/table"
-	"gocldf/db"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
-
-	_ "github.com/mattn/go-sqlite3"
 )
 
 type Dataset struct {
@@ -19,16 +18,16 @@ type Dataset struct {
 	Tables       map[string]*table.Table
 }
 
-func New(md_path string) *Dataset {
+func New(md_path string) (*Dataset, error) {
 	data, err := os.ReadFile(md_path)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 	var result map[string]interface{}
 
 	err = json.Unmarshal(data, &result)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
 
 	metadata := make(map[string]interface{}, len(result)-1)
@@ -44,24 +43,28 @@ func New(md_path string) *Dataset {
 		metadata,
 		make(map[string]*table.Table)}
 	for _, value := range result["tables"].([]interface{}) {
-		tbl := table.New(value.(map[string]interface{}))
+		tbl, err := table.New(value.(map[string]interface{}))
+		if err != nil {
+			return nil, err
+		}
 		res.Tables[tbl.CanonicalName] = tbl
 	}
-	return &res
+	return &res, nil
 }
 
-func (dataset *Dataset) LoadData() {
+func (dataset *Dataset) LoadData() error {
 	results := make(chan table.TableRead, len(dataset.Tables))
 	for _, tbl := range dataset.Tables {
 		go tbl.Read(filepath.Dir(dataset.MetadataPath), results)
 	}
 	for i := 0; i < len(dataset.Tables); i++ {
 		tableRead := <-results
 		if tableRead.Err != nil {
-			panic(tableRead.Err)
+			return tableRead.Err
 		}
 	}
 	close(results)
+	return nil
 }
 
 func (dataset *Dataset) UrlToTable() map[string]*table.Table {
@@ -80,29 +83,7 @@ func (dataset *Dataset) UrlToCanonicalName() map[string]string {
 	return res
 }
 
-func (dataset *Dataset) SqlSchema() string {
-	var (
-		res           []string
-		orderedTables = dataset.OrderedTables()
-		urlToName     = dataset.UrlToCanonicalName()
-		urlToTable    = dataset.UrlToTable()
-	)
-	for _, url := range orderedTables {
-		tbl, ok := dataset.Tables[urlToName[url]]
-		if ok {
-			res = append(res, tbl.SqlCreate(urlToTable))
-		}
-	}
-	for _, url := range orderedTables {
-		tbl, ok := dataset.Tables[urlToName[url]]
-		if ok {
-			res = append(res, tbl.SqlCreateAssociationTables(urlToTable))
-		}
-	}
-	return strings.Join(res, "\n")
-}
-
-func (dataset *Dataset) OrderedTables() []string {
+func (dataset *Dataset) orderedTables() (map[string]*table.Table, error) {
 	var urlToName = dataset.UrlToCanonicalName()
 	// Determine the order in which to create the tables
 	tables := []string{}
@@ -114,7 +95,7 @@ func (dataset *Dataset) OrderedTables() []string {
 	for len(tables) > 0 {
 		j++
 		if j > 100 {
-			panic("there may be cyclic dependencies between tables")
+			return nil, errors.New("there may be cyclic dependencies between tables")
 		}
 		// We loop over all tables that have not been ordered yet, trying to find one with
 		// only fks to already ordered tables.
@@ -138,45 +119,86 @@ func (dataset *Dataset) OrderedTables() []string {
 					break
 				}
 			} else {
-				panic("table not found")
+				return nil, errors.New("table not found")
 			}
 		}
 		if delIndex >= 0 {
 			tables = slices.Delete(tables, delIndex, delIndex+1)
 		}
 	}
-	return orderedTables
+	orderedTableMap := make(map[string]*table.Table, len(orderedTables))
+	for _, url := range orderedTables {
+		tbl, ok := dataset.Tables[urlToName[url]]
+		if ok {
+			orderedTableMap[url] = tbl
+		} else {
+			return orderedTableMap, fmt.Errorf("table %s not found", url)
+		}
+	}
+	return orderedTableMap, nil
 }
 
-func (dataset *Dataset) ToSqlite(db_path string) {
-	err := db.WithDatabase(db_path, func(database *sql.DB) error {
-		_, err := database.Exec(dataset.SqlSchema())
+func (dataset *Dataset) sqlSchema() (string, error) {
+	var (
+		res        []string
+		urlToTable = dataset.UrlToTable()
+	)
+	orderedTableMap, err := dataset.orderedTables()
+	if err != nil {
+		return "", err
+	}
+
+	for _, tbl := range orderedTableMap {
+		schema, err := tbl.SqlCreate(urlToTable)
 		if err != nil {
-			return err
+			return "", err
+		} else {
+			res = append(res, schema)
+		}
+	}
+	for _, tbl := range orderedTableMap {
+		for _, fk := range tbl.ManyToMany() {
+			res = append(res, tbl.SqlCreateAssociationTable(*fk, urlToTable))
 		}
+	}
+	return strings.Join(res, "\n"), nil
+}
 
-		orderedTables := dataset.OrderedTables()
-		urlToName := dataset.UrlToCanonicalName()
-		urlToTable := dataset.UrlToTable()
+type TableData struct {
+	TableName string
+	ColNames  []string
+	Rows      [][]any
+}
 
-		db.WithTransaction(database, func(tx *sql.Tx) {
-			for _, url := range orderedTables {
-				tbl, ok := dataset.Tables[urlToName[url]]
-				if ok {
-					tbl.SqlInsert(tx)
-				}
-			}
+// Function ToSqlite returns the data necessary to load the dataset into a SQLite database.
+func (dataset *Dataset) ToSqlite(tx *sql.Tx) (string, []TableData, error) {
+	var tableData = []TableData{}
 
-			for _, url := range orderedTables {
-				tbl, ok := dataset.Tables[urlToName[url]]
-				if ok {
-					tbl.SqlInsertAssociationTables(tx, urlToTable)
-				}
-			}
-		})
-		return err
-	}, true)
+	schema, err := dataset.sqlSchema()
+	if err != nil {
+		return "", tableData, err
+	}
+	orderedTables, err := dataset.orderedTables()
 	if err != nil {
-		return
+		return "", tableData, err
+	}
+	urlToTable := dataset.UrlToTable()
+
+	for _, tbl := range orderedTables {
+		rows, colNames, err := tbl.RowsToSql()
+		if err != nil {
+			return "", tableData, err
+		}
+		tableData = append(tableData, TableData{tbl.CanonicalName, colNames, rows})
+	}
+	for _, tbl := range orderedTables {
+		for _, fk := range tbl.ManyToMany() {
+			rows, tableName, colNames, err := tbl.AssociationTableRowsToSql(fk, urlToTable)
+			if err != nil {
+				return "", tableData, err
+			}
+			tableData = append(tableData, TableData{tableName, colNames, rows})
+		}
 	}
+	return schema, tableData, nil
 }

csvw/datatype/datatype.go

@@ -3,6 +3,8 @@ package datatype
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
+	"net/url"
 	"regexp"
 	"slices"
 	"strconv"
@@ -58,6 +60,7 @@ type BaseType struct {
 }
 
 var (
+	// Boolean: ToGo returns an object of type bool.
 	Boolean = BaseType{
 		GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
 			val, ok := dtProps["format"]
@@ -90,6 +93,7 @@ var (
 			return 0, nil
 		},
 	}
+	// String ------------------------------------------------------------------------------
 	String = BaseType{
 		GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
 			val, ok := dtProps["format"]
@@ -111,25 +115,38 @@ var (
 			return x.(string), nil
 		},
 		SqlType: "TEXT",
-		ToSql: func(dt *Datatype, x any) (any, error) {
-			return x.(string), nil
-		},
+		ToSql:   func(dt *Datatype, x any) (any, error) { return x.(string), nil },
 	}
+	// AnyURI: ToGo returns an object of type url.URL
 	AnyURI = BaseType{
 		GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
 			return map[string]any{}, nil
 		},
 		ToGo: func(dt *Datatype, s string) (any, error) {
-			return s, nil
+			u, err := url.Parse(s)
+			if err != nil {
+				return nil, err
+			}
+			// We don't want the rather lax parsing of url.Parse.
+			/*
+				_, err = url.ParseRequestURI(s)
+				if err != nil {
+					return nil, err
+				}
+			*/
+			return u, nil
 		},
 		ToString: func(dt *Datatype, x any) (string, error) {
-			return x.(string), nil
+			u := x.(*url.URL)
+			return u.String(), nil
 		},
 		SqlType: "TEXT",
 		ToSql: func(dt *Datatype, x any) (any, error) {
-			return x.(string), nil
+			u := x.(*url.URL)
+			return u.String(), nil
 		},
 	}
+	// Integer ----------------------------------------------------------------------------
 	Integer = BaseType{
 		GetDerivedDescription: func(dtProps map[string]any) (map[string]any, error) {
 			return map[string]any{}, nil
@@ -161,7 +178,7 @@ var (
 			return val, nil
 		},
 		ToString: func(dt *Datatype, x any) (string, error) {
-			return x.(string), nil
+			return fmt.Sprintf("%g", x.(float64)), nil
 		},
 		SqlType: "REAL",
 		ToSql: func(dt *Datatype, x any) (any, error) {
@@ -206,7 +223,7 @@ var BaseTypes = map[string]BaseType{
 	"json":    Json,
 }
 
-func New(jsonCol map[string]interface{}) *Datatype {
+func New(jsonCol map[string]interface{}) (*Datatype, error) {
 	base := "string"
 	dtProps := map[string]any{}
 
@@ -220,12 +237,11 @@ func New(jsonCol map[string]interface{}) *Datatype {
 			dtProps = val.(map[string]any)
 		}
 	}
-	//fmt.Println(base)
 	dd, err := BaseTypes[base].GetDerivedDescription(dtProps)
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	return &Datatype{Base: base, DerivedDescription: dd}
+	return &Datatype{Base: base, DerivedDescription: dd}, nil
 }
 
 func (dt *Datatype) ToString(val any) (string, error) {