add verb 'dupi like'

Author: scott-cotton

README.md

@@ -27,3 +27,7 @@ Find and query for repeated chunks of text.
 
 [Design Document](docs/design.md)
 
+## Library Reference
+
+
+

cmd/dupi/blot.go

@@ -1,3 +1,17 @@
+// Copyright 2021 the Dupi authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

cmd/dupi/dupi.go

@@ -33,7 +33,8 @@ var scMap = map[string]SubCmd{
 	"extract": newExtractCmd(),
 	"blot":    newBlotCmd(),
 	"unblot":  newUnblotCmd(),
-	"inspect": newInspectCmd()}
+	"inspect": newInspectCmd(),
+	"like":    newLikeCmd()}
 
 var gFlags = flag.NewFlagSet("dupi", flag.ExitOnError)
 

cmd/dupi/index.go

@@ -106,10 +106,6 @@ func (x *indexCmd) mkWalkFn(perr *error) func(path string, entry fs.DirEntry, er
 			return nil
 		}
 
-		if *x.verbose {
-			log.Printf("doPath %s\n", path)
-		}
-
 		f, e := os.Open(path)
 		if e != nil {
 			return e
@@ -121,6 +117,9 @@ func (x *indexCmd) mkWalkFn(perr *error) func(path string, entry fs.DirEntry, er
 			return err
 		}
 		doc := &dupi.Doc{Path: path, Dat: dat, End: uint32(len(dat))}
+		if *x.verbose {
+			log.Printf("indexing %s %d:%d\n", path, 0, doc.End)
+		}
 		return x.indexer.Add(doc)
 	}
 }

cmd/dupi/inspect.go

@@ -26,7 +26,8 @@ import (
 
 type inspectCmd struct {
 	subCmd
-	json *bool
+	json  *bool
+	files *bool
 }
 
 func newInspectCmd() *inspectCmd {
@@ -35,7 +36,8 @@ func newInspectCmd() *inspectCmd {
 		flags: flag.NewFlagSet("inspect", flag.ExitOnError)}
 	res := &inspectCmd{
 		subCmd: *sub,
-		json:   sub.flags.Bool("json", false, "output json.")}
+		json:   sub.flags.Bool("json", false, "output json."),
+		files:  sub.flags.Bool("files", false, "output file info.")}
 	return res
 }
 

cmd/dupi/like.go

@@ -0,0 +1,139 @@
+// Copyright 2021 the Dupi authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"sort"
+
+	"github.com/go-air/dupi"
+	"github.com/go-air/dupi/token"
+)
+
+type likeCmd struct {
+	subCmd
+}
+
+func newLikeCmd() *likeCmd {
+	sc := &subCmd{flags: flag.NewFlagSet("like", flag.ExitOnError)}
+	lc := &likeCmd{subCmd: *sc}
+	return lc
+}
+
+func (lc *likeCmd) Usage() string {
+	return "[file path]"
+}
+
+func (lc *likeCmd) Run(args []string) error {
+	lc.flags.Parse(args)
+	root := getIndexRoot()
+	idx, err := dupi.OpenIndex(root)
+	if err != nil {
+		log.Fatalf("couldn't open dupi index at '%s': %s", root, err)
+	}
+	defer idx.Close()
+	for _, fname := range args {
+		if err := doFilename(idx, fname); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type docKey struct {
+	Path       string
+	start, end uint32
+}
+
+func dkey(doc *dupi.Doc) docKey {
+	return docKey{
+		Path:  doc.Path,
+		start: doc.Start,
+		end:   doc.End}
+}
+
+func doFilename(idx *dupi.Index, fname string) error {
+	f, e := os.Open(fname)
+	if e != nil {
+		return e
+	}
+	defer f.Close()
+	dat, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+	doc := &dupi.Doc{Path: fname, Dat: dat, End: uint32(len(dat))}
+	blots := idx.BlotDoc(nil, doc)
+	var toks []token.T
+	toks = idx.TokenFunc()(nil, doc.Dat, 0)
+	j := 0
+	for i := range toks {
+		if toks[i].Tag != token.Word {
+			continue
+		}
+		toks[j] = toks[i]
+		j++
+	}
+	N := uint32(idx.NumShards()) * (1 << 16)
+	seqLen := idx.SeqLen()
+	bm := make(map[uint32][]byte, len(blots))
+	for i, b := range blots {
+		beg := toks[i].Pos
+		end := toks[i+seqLen].Pos + uint32(len(toks[i+seqLen].Lit))
+		bm[b%N] = dat[beg:end]
+	}
+	query := idx.StartQuery(dupi.QueryMaxBlot)
+	var db dupi.Blot
+	found := make(map[docKey]int)
+	for _, blot := range blots {
+		b := blot % N
+		db.Blot = b
+		db.Docs = nil
+		if err := query.Get(&db); err != nil {
+			return err
+		}
+		for i := range db.Docs {
+			doc := &db.Docs[i]
+			dk := dkey(doc)
+			if found[dk] != 0 {
+				continue
+			}
+			rm, err := idx.FindBlots(bm, doc)
+			if err != nil {
+				return err
+			}
+			if len(rm) == 0 {
+				continue
+			}
+			found[dk]++
+		}
+	}
+	keys := make([]docKey, 0, len(found))
+	for k, _ := range found {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		return found[keys[i]] < found[keys[j]]
+	})
+	fmt.Printf("like %s:\n", fname)
+	for _, dk := range keys {
+		fmt.Printf("\t%s %d:%d\n", dk.Path, dk.start, dk.end)
+	}
+	return nil
+}

cmd/dupi/unblot.go

@@ -1,3 +1,17 @@
+// Copyright 2021 the Dupi authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

dmd/adder_test.go

@@ -1,3 +1,17 @@
+// Copyright 2021 the Dupi authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package dmd
 
 import (

doc.go

@@ -62,7 +62,7 @@ func (doc *Doc) Load() error {
 		doc.Dat = make([]byte, doc.End-doc.Start)
 		_, err = f.ReadAt(doc.Dat, int64(doc.Start))
 		if err != nil {
-			return fmt.Errorf("readat len=%d at=%d: %w\n", len(doc.Dat), doc.Start, err)
+			return fmt.Errorf("readat %s len=%d at=%d: %w\n", doc.Path, len(doc.Dat), doc.Start, err)
 		}
 		f.Close()
 	}

index.go

@@ -15,6 +15,7 @@
 package dupi
 
 import (
+	"bytes"
 	"fmt"
 	"log"
 	"math"
@@ -188,6 +189,47 @@ func (x *Index) JoinBlot(shard uint32, sblot uint16) uint32 {
 	return blot
 }
 
+func (x *Index) FindBlots(m map[uint32][]byte, doc *Doc) (map[uint32][]byte, error) {
+	if doc.Dat == nil {
+		err := doc.Load()
+		if err != nil {
+			return nil, err
+		}
+	}
+	toks := x.TokenFunc()(nil, doc.Dat, doc.Start)
+	j := 0
+	for _, tok := range toks {
+		if tok.Tag != token.Word {
+			continue
+		}
+		toks[j] = tok
+		j++
+	}
+	blotter := x.Blotter()
+	seqLen := x.SeqLen()
+	nShard := uint32(x.NumShards())
+	res := make(map[uint32][]byte)
+	for i, tok := range toks[:j] {
+		blot := blotter.Blot(tok.Lit)
+		if i < seqLen {
+			continue
+		}
+		blot %= nShard * (1 << 16)
+		if m[blot] == nil {
+			continue
+		}
+		start := toks[i-seqLen].Pos
+		end := tok.Pos + uint32(len(tok.Lit))
+		txt := doc.Dat[start:end]
+		if bytes.Equal(m[blot], txt) {
+			res[blot] = txt
+		} else {
+			//fmt.Printf("not equal: %s %s\n", string(m[blot]), string(txt))
+		}
+	}
+	return res, nil
+}
+
 func (x *Index) FindBlot(theBlot uint32, doc *Doc) (start, end uint32, err error) {
 	if doc.Dat == nil {
 		err = doc.Load()