feat(server): add tool to migrate uc v1 data and upload assets

Author: rot1024

Diff for: go.work.sum

@@ -470,6 +470,7 @@ google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
 gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=
 honnef.co/go/js/dom v0.0.0-20210725211120-f030747120f2 h1:oomkgU6VaQDsV6qZby2uz1Lap0eXmku8+2em3A/l700=
 honnef.co/go/js/dom v0.0.0-20210725211120-f030747120f2/go.mod h1:sUMDUKNB2ZcVjt92UnLy3cdGs+wDAcrPdV3JP6sVgA4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
 honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
 rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
 rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY=

Diff for: server/main.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/eukarya-inc/reearth-plateauview/server/putil"
+	"github.com/eukarya-inc/reearth-plateauview/server/tool"
 	"github.com/go-playground/validator/v10"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
@@ -27,7 +28,10 @@ func main() {
 	conf := lo.Must(NewConfig())
 
 	if len(os.Args) > 1 && os.Args[1] != "" {
-		tool(conf, os.Args[1:])
+		tool.Main(&tool.Config{
+			CMS_BaseURL: conf.CMS_BaseURL,
+			CMS_Token:   conf.CMS_Token,
+		}, os.Args[1:])
 		return
 	}
 

Diff for: server/tool/migrate-v1.go

@@ -0,0 +1,196 @@
+package tool
+
+import (
+	"archive/zip"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+func migrateV1(conf *Config, args []string) error {
+	var listFilePath, output, baseURL string
+	var offset int
+	var wetrun bool
+
+	flags := flag.NewFlagSet("migrate-v1", flag.ExitOnError)
+	flags.StringVar(&listFilePath, "list", "", "list file path")
+	flags.StringVar(&output, "output", "", "output file path")
+	flags.StringVar(&baseURL, "base", "", "base URL")
+	flags.BoolVar(&wetrun, "wetrun", false, "wet run")
+	flags.IntVar(&offset, "offset", 0, "offset")
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+
+	if baseURL == "" {
+		return fmt.Errorf("base URL is required")
+	}
+
+	if output != "" {
+		_ = os.MkdirAll(output, 0755)
+	}
+
+	listFile, err := os.ReadFile(listFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to read list file: %w", err)
+	}
+
+	list, err := parseList(listFile)
+
+	if offset > 0 {
+		if len(list) < offset {
+			return fmt.Errorf("offset is too large")
+		}
+		list = list[offset:]
+	}
+
+	le := len(list)
+	if err != nil {
+		return fmt.Errorf("failed to parse list: %w", err)
+	}
+
+	var file *os.File
+	var zw *zip.Writer
+
+	exhangeZw := func(g string) error {
+		if zw != nil {
+			if err := zw.Close(); err != nil {
+				return fmt.Errorf("failed to close zip: %w", err)
+			}
+		}
+
+		if file != nil {
+			if err := file.Close(); err != nil {
+				return fmt.Errorf("failed to close file: %w", err)
+			}
+		}
+
+		if g == "" {
+			zw = nil
+			file = nil
+			return nil
+		}
+
+		file, err = os.OpenFile(filepath.Join(output, g+".zip"), os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return fmt.Errorf("failed to open file: %w", err)
+		}
+
+		zw = zip.NewWriter(file)
+		return nil
+	}
+
+	defer func() {
+		if zw != nil {
+			_ = zw.Close()
+		}
+		if file != nil {
+			_ = file.Close()
+		}
+	}()
+
+	group := ""
+	for i, p := range list {
+		g := p[1]
+		filePath := path.Join(p[2:]...)
+		path := filepath.Join(p...)
+		u, err := url.JoinPath(baseURL, path)
+		if err != nil {
+			return fmt.Errorf("failed to join path: %w", err)
+		}
+
+		fmt.Printf("%d/%d | %s | %s | %s\n", i+1, le, g, filePath, u)
+
+		if group != g {
+			fmt.Printf("group: %s\n", g)
+
+			if wetrun {
+				if err := exhangeZw(g); err != nil {
+					return err
+				}
+			}
+
+			group = g
+		}
+
+		if wetrun {
+			if err := downloadAndAddToZip(zw, path, u, filePath); err != nil {
+				return fmt.Errorf("failed to download: %w", err)
+			}
+		}
+	}
+
+	if err := exhangeZw(""); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func downloadAndAddToZip(zw *zip.Writer, path, url, filePath string) error {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to request: %w", err)
+	}
+
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to request: %s", res.Status)
+	}
+
+	f, err := zw.Create(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to create zip entry: %w", err)
+	}
+
+	if _, err := io.Copy(f, res.Body); err != nil {
+		return fmt.Errorf("failed to copy: %w", err)
+	}
+
+	return nil
+}
+
+func parseList(b []byte) (res [][]string, err error) {
+	start := 0
+	for i, c := range b {
+		if c == '\n' {
+			r := string(b[start:i])
+			start = i + 1
+
+			s := reSpace.Split(r, -1)
+			if len(s) <= 3 {
+				err = fmt.Errorf("invalid line at %d: %s", i, r)
+				return
+			}
+
+			t := strings.Split(s[3], "/")
+			if strings.HasPrefix(t[len(t)-1], ".") {
+				continue
+			}
+
+			if len(t) <= 2 {
+				err = fmt.Errorf("invalid line at %d: %s", i, r)
+				return
+			}
+
+			res = append(res, t)
+		}
+	}
+
+	return
+}
+
+var reSpace = regexp.MustCompile(`\s+`)

Diff for: server/tool.go renamed to server/tool/setup-city-items.go

@@ -1,4 +1,4 @@
-package main
+package tool
 
 import (
 	"context"
@@ -13,30 +13,6 @@ import (
 	"github.com/samber/lo"
 )
 
-func tool(conf *Config, args []string) {
-	subommand := args[0]
-	var err error
-
-	switch subommand {
-	case "setup-city-items":
-		err = setupCityItems(conf, args[1:])
-	case "help":
-		err = help(conf)
-	default:
-		err = errors.New("invalid subcommand")
-	}
-
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}
-
-func help(*Config) error {
-	fmt.Println(`Usage: plateauview <command> [arguments] [options] [flags]`)
-	return nil
-}
-
 func setupCityItems(conf *Config, args []string) error {
 	println("setup-city-items")
 

Diff for: server/tool/tool.go

@@ -0,0 +1,40 @@
+package tool
+
+import (
+	"errors"
+	"fmt"
+	"os"
+)
+
+type Config struct {
+	CMS_BaseURL string
+	CMS_Token   string
+}
+
+func Main(conf *Config, args []string) {
+	subommand := args[0]
+	var err error
+
+	switch subommand {
+	case "setup-city-items":
+		err = setupCityItems(conf, args[1:])
+	case "migrate-v1":
+		err = migrateV1(conf, args[1:])
+	case "upload-assets":
+		err = uploadAssets(conf, args[1:])
+	case "help":
+		err = help(conf)
+	default:
+		err = errors.New("invalid subcommand")
+	}
+
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func help(*Config) error {
+	fmt.Println(`Usage: plateauview <command> [arguments] [options] [flags]`)
+	return nil
+}

Diff for: server/tool/upload-assets.go

@@ -0,0 +1,69 @@
+package tool
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+
+	cms "github.com/reearth/reearth-cms-api/go"
+)
+
+func uploadAssets(conf *Config, args []string) error {
+	var base, token, pid, target string
+
+	flags := flag.NewFlagSet("upload-assets", flag.ExitOnError)
+	flags.StringVar(&base, "base", conf.CMS_BaseURL, "CMS base URL")
+	flags.StringVar(&token, "token", conf.CMS_Token, "CMS token")
+	flags.StringVar(&pid, "project", "", "project ID")
+	flags.StringVar(&target, "target", "", "target dir path")
+
+	if err := flags.Parse(args); err != nil {
+		return err
+	}
+
+	if pid == "" {
+		return errors.New("project ID is required")
+	}
+
+	c, err := cms.New(base, token)
+	if err != nil {
+		return err
+	}
+
+	files, err := os.ReadDir(target)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+
+		ext := path.Ext(f.Name())
+		if ext != ".zip" {
+			continue
+		}
+
+		file, err := os.Open(filepath.Join(target, f.Name()))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("%s -> ", file.Name())
+
+		assetID, err := c.UploadAssetDirectly(ctx, pid, file.Name(), file)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("%s\n", assetID)
+	}
+
+	return nil
+}

Diff for: tools/src/args.rs

@@ -23,13 +23,6 @@ pub enum Commands {
         #[clap(short, long)]
         output: Option<PathBuf>,
     },
-    /// PLATEAU VIEW 1.1 のS3内に格納されたユースケースデータのファイルを移行します。
-    MigrateUC {
-        #[clap(short, long)]
-        list_path: PathBuf,
-        #[clap(short, long)]
-        output: Option<PathBuf>,
-    },
 }
 
 #[derive(Debug, Clone, ValueEnum)]