init

Author: fumiama

.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: 稳定版
+on:
+  push:
+    tags:
+      - v*
+
+env:
+  GITHUB_TOKEN: ${{ github.token }}
+
+jobs:
+  my-job:
+    name: Build gocq-sqlite3-migrate on Push Tag 🚀
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.19
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Cache Go
+        id: cache
+        uses: actions/cache@v2
+        with:
+          # A list of files, directories, and wildcard patterns to cache and restore
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-build-${{ hashFiles('**/go.sum') }}
+
+      - name: Tidy Go modules
+        run: go mod tidy
+
+      - name: Build linux-x64
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/gocq-sqlite3-migrate-linux-x64
+      - name: Build linux-x86
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -ldflags="-s -w" -o artifacts/gocq-sqlite3-migrate-linux-x86
+      - name: Build windows-x64
+        run: CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/gocq-sqlite3-migrate-x64.exe
+      - name: Build windows-x86
+        run: CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -ldflags="-s -w" -o artifacts/gocq-sqlite3-migrate-x86.exe
+      - name: Build arm64
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 GOARM=7 go build -ldflags="-s -w" -o artifacts/gocq-sqlite3-migrate-linux-arm64
+      - name: Build armv6
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w" -o artifacts/gocq-sqlite3-migrate-linux-armv6
+      - name: Build darwin amd64
+        run: CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o artifacts/gocq-sqlite3-migrate-darwin-amd64
+      - name: Build darwin arm64
+        run: CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags="-s -w" -o artifacts/gocq-sqlite3-migrate-darwin-arm64
+
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/gocq-sqlite3-migrate-*
+          tag: ${{ github.ref }}
+          overwrite: true
+          file_glob: true

.gitignore

@@ -13,3 +13,6 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+/data
+go.sum

README.md

@@ -1,2 +1,20 @@
 # gocq-sqlite3-migrate
 go-cqhttp leveldb v3 到 sqlite3 迁移工具
+
+## 安装
+
+你可以下载 [已经编译好的二进制文件](https://github.com/RomiChan/gocq-sqlite3-migrate/releases).
+
+从源码安装:
+```bash
+$ go install github.com/RomiChan/gocq-sqlite3-migrate@latest
+```
+
+## 使用方法
+
+```bash
+./gocq-sqlite3-migrate -from xxx -to yyy
+```
+默认值：
+ * from: `data/leveldb-v3`
+ * to: `data/sqlite3/msg.db`

go.mod

@@ -0,0 +1,35 @@
+module github.com/RomiChan/gocq-sqlite3-migrate
+
+go 1.19
+
+require (
+	github.com/FloatTech/sqlite v0.5.0
+	github.com/Mrs4s/MiraiGo v0.0.0-20220828090150-a3c348100dfe
+	github.com/Mrs4s/go-cqhttp v1.0.0-rc3.0.20221109121432-fa267b6a2d47
+	github.com/pkg/errors v0.9.1
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+)
+
+require (
+	github.com/FloatTech/ttl v0.0.0-20220715042055-15612be72f5b // indirect
+	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/fumiama/go-base16384 v1.5.2 // indirect
+	github.com/fumiama/sqlite3 v1.14.6 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	github.com/segmentio/asm v1.1.3 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/tidwall/gjson v1.14.3 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.14.6 // indirect
+	modernc.org/mathutil v1.4.1 // indirect
+	modernc.org/memory v1.0.5 // indirect
+)

leveldb/const.go

@@ -0,0 +1,25 @@
+package leveldb
+
+const dataVersion = 1
+
+const (
+	group        = 0x0
+	private      = 0x1
+	guildChannel = 0x2
+)
+
+type coder byte
+
+const (
+	coderNil coder = iota
+	coderInt
+	coderUint
+	coderInt32
+	coderUint32
+	coderInt64
+	coderUint64
+	coderString
+	coderMSG      // global.MSG
+	coderArrayMSG // []global.MSG
+	coderStruct   // struct{}
+)

leveldb/leveldb.go

@@ -0,0 +1,66 @@
+package leveldb
+
+import (
+	"strconv"
+
+	"github.com/Mrs4s/MiraiGo/utils"
+	"github.com/pkg/errors"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+type Database struct {
+	db *leveldb.DB
+}
+
+func Open(dbpath string) (ldb *Database, err error) {
+	d, err := leveldb.OpenFile(dbpath, &opt.Options{
+		WriteBuffer: 32 * opt.KiB,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "open leveldb error")
+	}
+	return &Database{db: d}, nil
+}
+
+func (ldb *Database) Close() error {
+	return ldb.db.Close()
+}
+
+func (ldb *Database) ForEach(f func(x any) error) (errs []error) {
+	iter := ldb.db.NewIterator(nil, nil)
+	for iter.Next() {
+		value := iter.Value()
+		if len(value) == 0 {
+			continue
+		}
+		r, err := newReader(utils.B2S(value))
+		if err != nil {
+			errs = append(errs, errors.Wrap(err, "new reader failed"))
+			continue
+		}
+		flg := r.uvarint()
+		switch flg {
+		case group:
+			err = f(r.readStoredGroupMessage())
+			if err != nil {
+				errs = append(errs, errors.Wrap(err, "decode group message callback failed"))
+			}
+		case private:
+			err = f(r.readStoredPrivateMessage())
+			if err != nil {
+				errs = append(errs, errors.Wrap(err, "decode private message callback failed"))
+			}
+		case guildChannel:
+			err = f(r.readStoredGuildChannelMessage())
+			if err != nil {
+				errs = append(errs, errors.Wrap(err, "decode guild channel message callback failed"))
+			}
+		default:
+			errs = append(errs, errors.New("unknown message flag "+strconv.Itoa(int(flg))))
+		}
+
+	}
+	iter.Release()
+	return
+}

leveldb/reader.go

@@ -0,0 +1,131 @@
+package leveldb
+
+import (
+	"encoding/binary"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/Mrs4s/go-cqhttp/global"
+)
+
+type intReader struct {
+	data string
+	*strings.Reader
+}
+
+func newIntReader(s string) intReader {
+	return intReader{
+		data:   s,
+		Reader: strings.NewReader(s),
+	}
+}
+
+func (r *intReader) varint() int64 {
+	i, _ := binary.ReadVarint(r)
+	return i
+}
+
+func (r *intReader) uvarint() uint64 {
+	i, _ := binary.ReadUvarint(r)
+	return i
+}
+
+// reader implements the index read.
+// data format is the same as the writer's
+type reader struct {
+	data        intReader
+	strings     intReader
+	stringIndex map[uint64]string
+}
+
+func (r *reader) coder() coder    { o, _ := r.data.ReadByte(); return coder(o) }
+func (r *reader) varint() int64   { return r.data.varint() }
+func (r *reader) uvarint() uint64 { return r.data.uvarint() }
+func (r *reader) int32() int32    { return int32(r.varint()) }
+func (r *reader) int64() int64    { return r.varint() }
+func (r *reader) uint64() uint64  { return r.uvarint() }
+
+// func (r *reader) uint32() uint32 { return uint32(r.uvarint()) }
+// func (r *reader) int() int        { return int(r.varint()) }
+// func (r *reader) uint() uint      { return uint(r.uvarint()) }
+
+func (r *reader) string() string {
+	off := r.data.uvarint()
+	if s, ok := r.stringIndex[off]; ok {
+		return s
+	}
+	_, _ = r.strings.Seek(int64(off), io.SeekStart)
+	l := int64(r.strings.uvarint())
+	whence, _ := r.strings.Seek(0, io.SeekCurrent)
+	s := r.strings.data[whence : whence+l]
+	r.stringIndex[off] = s
+	return s
+}
+
+func (r *reader) msg() global.MSG {
+	length := r.uvarint()
+	msg := make(global.MSG, length)
+	for i := uint64(0); i < length; i++ {
+		s := r.string()
+		msg[s] = r.obj()
+	}
+	return msg
+}
+
+func (r *reader) arrayMsg() []global.MSG {
+	length := r.uvarint()
+	msgs := make([]global.MSG, length)
+	for i := range msgs {
+		msgs[i] = r.msg()
+	}
+	return msgs
+}
+
+func (r *reader) obj() interface{} {
+	switch coder := r.coder(); coder {
+	case coderNil:
+		return nil
+	case coderInt:
+		return int(r.varint())
+	case coderUint:
+		return uint(r.uvarint())
+	case coderInt32:
+		return int32(r.varint())
+	case coderUint32:
+		return uint32(r.uvarint())
+	case coderInt64:
+		return r.varint()
+	case coderUint64:
+		return r.uvarint()
+	case coderString:
+		return r.string()
+	case coderMSG:
+		return r.msg()
+	case coderArrayMSG:
+		return r.arrayMsg()
+	default:
+		panic("db/leveldb: invalid coder " + strconv.Itoa(int(coder)))
+	}
+}
+
+func newReader(data string) (*reader, error) {
+	in := newIntReader(data)
+	v := in.uvarint()
+	if v != dataVersion {
+		return nil, errors.Errorf("db/leveldb: invalid data version %d", v)
+	}
+	sl := int64(in.uvarint())
+	dl := int64(in.uvarint())
+	whence, _ := in.Seek(0, io.SeekCurrent)
+	sData := data[whence : whence+sl]
+	dData := data[whence+sl : whence+sl+dl]
+	r := reader{
+		data:        newIntReader(dData),
+		strings:     newIntReader(sData),
+		stringIndex: make(map[uint64]string),
+	}
+	return &r, nil
+}