refactor, cleanup errors and document loading

Author: scott-cotton

cmd/dupi/unblot.go

@@ -43,16 +43,20 @@ func (ub *unblotCmd) Run(args []string) error {
 		m := make(map[string][]*dupi.Doc)
 		for i := range blot.Docs {
 			doc := &blot.Docs[i]
+
 			start, end, err := idx.FindBlot(hex, doc)
 			if err != nil {
 				log.Printf("warning: %s", err)
 				continue
 			}
-			dat := string(doc.Dat[start:end])
+			dat := string(doc.Dat[start-doc.Start : end-doc.Start])
 			doc.Dat = nil
 			m[dat] = append(m[dat], doc)
 		}
 		for k, ds := range m {
+			if len(ds) < 2 {
+				continue
+			}
 			fmt.Printf("text:\n'''\n%s'''\n", k)
 			for _, d := range ds {
 				fmt.Printf("\t%s %d:%d\n", d.Path, d.Start, d.End)

doc.go

@@ -16,6 +16,13 @@
 // data in large sets of documents.
 package dupi
 
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+)
+
 type Doc struct {
 	Path  string
 	Start uint32
@@ -28,3 +35,36 @@ func NewDoc(path, body string) *Doc {
 		Path: path,
 		Dat:  []byte(body)}
 }
+
+func (doc *Doc) Load() error {
+	var (
+		f   *os.File
+		err error
+	)
+
+	f, err = os.Open(doc.Path)
+	if err != nil {
+		return err
+	}
+
+	if doc.Start == 0 && doc.End == 0 {
+		doc.Dat, err = ioutil.ReadAll(f)
+		if err != nil {
+			return fmt.Errorf("readall: %w", err)
+		}
+		f.Close()
+		doc.End = uint32(len(doc.Dat))
+	} else {
+		_, err = f.Seek(int64(doc.Start), io.SeekStart)
+		if err != nil {
+			return fmt.Errorf("seek: %w", err)
+		}
+		doc.Dat = make([]byte, doc.End-doc.Start)
+		_, err = f.ReadAt(doc.Dat, int64(doc.Start))
+		if err != nil {
+			return fmt.Errorf("readat len=%d at=%d: %w\n", len(doc.Dat), doc.Start, err)
+		}
+		f.Close()
+	}
+	return nil
+}

index.go

@@ -16,8 +16,6 @@ package dupi
 
 import (
 	"fmt"
-	"io"
-	"io/ioutil"
 	"log"
 	"os"
 
@@ -166,17 +164,10 @@ func (x *Index) JoinBlot(shard uint32, sblot uint16) uint32 {
 
 func (x *Index) FindBlot(theBlot uint32, doc *Doc) (start, end uint32, err error) {
 	if doc.Dat == nil {
-		var f *os.File
-		f, err = os.Open(doc.Path)
+		err = doc.Load()
 		if err != nil {
 			return
 		}
-		doc.Dat, err = ioutil.ReadAll(f)
-		if err != nil {
-			return
-		}
-		f.Close()
-		doc.Dat = doc.Dat[doc.Start:doc.End]
 	}
 	toks := x.TokenFunc()(nil, doc.Dat, doc.Start)
 	j := 0
@@ -203,7 +194,7 @@ func (x *Index) FindBlot(theBlot uint32, doc *Doc) (start, end uint32, err error
 		end = tok.Pos + uint32(len(tok.Lit))
 		return
 	}
-	err = io.EOF
+	err = fmt.Errorf("blot %x not found", theBlot)
 	return
 }