chore: init

Author: ndabAP

.gitignore

@@ -0,0 +1,3 @@
+.env
+.vscode
+__debug_bin*

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Julian Claus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,24 @@
+GOLANGCILINT_VERSION := v2.5.0
+GO_PATH := $(shell go env GOPATH)/bin
+GOLANGCILINT_BIN := $(GO_PATH)/golangci-lint
+
+.PHONY: lint fmt test docs
+
+all: lint fmt test
+
+$(GOLANGCILINT_BIN):
+	@if ! test -x $(GOLANGCILINT_BIN) || ! $(GOLANGCILINT_BIN) --version | grep -q $(GOLANGCILINT_VERSION); then \
+		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GO_PATH) $(GOLANGCILINT_VERSION); \
+	fi
+
+fmt: $(GOLANGCILINT_BIN)
+	$(GOLANGCILINT_BIN) fmt ./... -v
+
+lint: $(GOLANGCILINT_BIN)
+	@$(GOLANGCILINT_BIN) run $(LINT_FLAGS) ./... --fix -v
+
+test:
+	go test -v ./... -short
+
+docs:
+	cd web && npm run build

README.md

@@ -0,0 +1,24 @@
+# entityscrape
+
+`entityscrape` is a live demo of the Go module
+[entitydebs](https://github.com/ndabAP/entitydebs), a social science tool to
+programmatically analyze entities in non-fictional texts. In particular, it's
+well-suited to extract the sentiment for an entity using dependency parsing.
+Tokenization is highly customizable and supports the Google Cloud Natural
+Language API out-of-the-box.
+
+The live demo gives insights to the following questions:
+
+- How do politicians describe their country in governmental speeches?
+- Which current topics correlate with celebrities?
+- What are the most common root verbs used in different music genres?
+
+Visit the [live demo](https://ndabap.github.io/entityscrape/).
+
+## Author
+
+[Julian Claus](https://www.julian-claus.de) and contributors.
+
+## License
+
+MIT

cases/conduct.go

@@ -0,0 +1,210 @@
+package cases
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"errors"
+	"log/slog"
+	"math/rand/v2"
+	"os"
+	"strings"
+	"unicode"
+
+	"github.com/ndabAP/entitydebs"
+	"github.com/ndabAP/entitydebs/tokenize"
+	"github.com/ndabAP/entitydebs/tokenize/nlp"
+	"github.com/ndabAP/entityscrape/sbd"
+	"github.com/ndabAP/entityscrape/translator"
+	"golang.org/x/text/language"
+)
+
+func (study study[samples, aggregated]) Conduct(ctx context.Context) error {
+	slog.Debug("processing subjects", "n", len(study.Subjects))
+
+	translator := translator.NewGoogle(ctx, GoogleCloudSvcAccountKey)
+	for subject, analyses := range study.Subjects {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		slog.Debug("processing analyses", "subject", subject)
+		var (
+			entity        = analyses.Entity
+			ext           = analyses.Ext
+			feats         = analyses.Feats
+			filenames     = analyses.Filenames
+			fuzzyMatching = analyses.FuzzyMatching
+			lang          = analyses.Language
+			parser        = analyses.Parser
+		)
+		tokenizer := nlp.New(GoogleCloudSvcAccountKey, lang.String())
+		frames, err := study.frames(
+			ctx,
+			entity,
+			filenames,
+			parser,
+			fuzzyMatching,
+			tokenizer,
+			feats,
+			lang,
+		)
+		if err != nil {
+			return err
+		}
+		slog.Debug("analysis done")
+
+		slog.Debug("collecting samples")
+		samples := study.collect(frames)
+		slog.Debug("sample collection done")
+
+		slog.Debug("aggregating samples")
+		aggregated := study.aggregate(samples)
+		slog.Debug("aggregation done")
+
+		slog.Debug("reporting aggregation")
+		translator := func(w []string) ([]string, error) {
+			switch lang {
+			case language.English:
+				slog.Debug("skipping translation for English")
+				return w, nil
+			default:
+			}
+
+			return translator.Translate(w, lang, language.English)
+		}
+		if err := func() error {
+			pref := strings.Map(func(r rune) rune {
+				switch {
+				case r >= 'a' && r <= 'z':
+					return r
+				case r >= 'A' && r <= 'Z':
+					return unicode.ToLower(r)
+				case r == ' ', r == '-':
+					return '_'
+				default:
+					return -1
+				}
+			}, subject)
+			writer, err := study.store.NewWriter(pref, ext)
+			if err != nil {
+				return err
+			}
+			//nolint:errcheck
+			defer writer.Close()
+
+			if err := study.report(aggregated, translator, writer); err != nil {
+				return err
+			}
+
+			return nil
+		}(); err != nil {
+			return err
+		}
+		slog.Debug("reporting done", "subject", subject)
+	}
+
+	return nil
+}
+
+func (study study[samples, aggregated]) frames(
+	ctx context.Context,
+	entity,
+	filenames []string,
+	parser Parser,
+	fuzzyMatching bool,
+	tokenizer tokenize.Tokenizer,
+	feats tokenize.Features,
+	lang language.Tag,
+) (
+	entitydebs.Frames,
+	error,
+) {
+	slog.Debug("parsing files", "n", len(filenames))
+	if fuzzyMatching {
+		slog.Debug("fuzzy matching enabled")
+	}
+
+	var (
+		texts = make([]string, 0, len(filenames))
+
+		textChan = make(chan []byte, 50)
+		errChan  = make(chan error, 1)
+	)
+
+	// Consumer
+	go func() {
+		defer close(errChan)
+
+		for text := range textChan {
+			n := rand.Uint64N(100)
+			if n >= SampleRate {
+				continue
+			}
+
+			t := string(text)
+
+			if fuzzyMatching {
+				var (
+					buf = new(bytes.Buffer)
+
+					c    = make(chan string, 50)
+					done = make(chan struct{}, 1)
+				)
+
+				// Consumer
+				study.fuzzyMatch(c, entity, buf, done)
+				// Producer
+				sbd.Tokenize(lang, t, c)
+				close(c)
+
+				<-done
+				if buf.Len() > 0 {
+					texts = append(texts, strings.TrimSpace(buf.String()))
+				}
+
+				continue
+			}
+
+			// No fuzzy matching.
+			texts = append(texts, strings.TrimSpace(t))
+		}
+	}()
+	// Producer
+	go func() {
+		defer close(textChan)
+
+		for _, filename := range filenames {
+			file, err := os.Open(filename)
+			if err != nil {
+				errChan <- err
+				return
+			}
+			for err := range parser(file, textChan) {
+				if errors.Is(err, bufio.ErrTooLong) {
+					continue
+				}
+				errChan <- err
+			}
+			//nolint:errcheck
+			_ = file.Close()
+		}
+	}()
+
+	select {
+	case <-ctx.Done():
+		return entitydebs.Frames{}, ctx.Err()
+
+	case err := <-errChan:
+		if err != nil {
+			return entitydebs.Frames{}, err
+		}
+	}
+	slog.Debug("texts sampled and parsed", "n", len(texts))
+
+	slog.Debug("generating frames")
+	src := entitydebs.NewSource(entity, texts)
+	return src.Frames(ctx, tokenizer, feats, entitydebs.NFKC)
+}

cases/corpus.go

@@ -0,0 +1,35 @@
+package cases
+
+import (
+	"io/fs"
+	"path/filepath"
+)
+
+// corpusDir is the absolute directory to the corpus.
+var corpusDir string
+
+func SetCorpusRootDir(base string) {
+	corpusDir = filepath.Join(base, "corpus")
+}
+
+func GetCorpusRootDir() string {
+	return corpusDir
+}
+
+func WalkCorpus(corpus string, fn func(filename string) error) error {
+	root := filepath.Join(corpusDir, corpus)
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		switch filepath.Ext(path) {
+		case ".gitignore", ".gitkeep":
+			return nil
+		}
+		return fn(path)
+	})
+	return err
+}

cases/fuzzy_match.go

@@ -0,0 +1,44 @@
+package cases
+
+import (
+	"bytes"
+	"strings"
+)
+
+// fuzzyMatch reads sentences from channel c, applies fuzzy matching against
+// entity, and writes matching sentences into buf. When done, it signals
+// completion by closing the done channel.
+func (study study[samples, aggregated]) fuzzyMatch(
+	c chan string,
+	entity []string,
+	buf *bytes.Buffer,
+	done chan struct{},
+) {
+	go func() {
+		defer close(done)
+
+		for s := range c {
+			// Size
+			if len(s) < 3 {
+				continue
+			}
+			// Filters
+			switch {
+			case strings.Contains(s, "http"), strings.Contains(s, ".com"), strings.Contains(s, "www"):
+				continue
+			default:
+			}
+
+			// Entity
+			for _, e := range entity {
+				if !strings.Contains(s, e) {
+					continue
+				}
+
+				// Concatenate sentence.
+				buf.WriteString(s)
+				break
+			}
+		}
+	}()
+}