[refactor] simplify Data Import script with MobX-RESTful-migrator

package.json

@@ -16,6 +16,7 @@
     "@strapi/strapi": "~5.23.5",
     "better-sqlite3": "^12.2.0",
     "cross-env": "^10.0.0",
+    "mobx-restful-migrator": "^0.1.0",
     "pg": "^8.16.3",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",

scripts/import-data-refactored.ts

@@ -0,0 +1,125 @@
+#!/usr/bin/env tsx
+
+/**
+ * Refactored Strapi database import script using MobX-RESTful-migrator
+ */
+
+import * as fs from 'node:fs';
+import { RestMigrator, ConsoleLogger } from 'mobx-restful-migrator';
+
+import { ExcelReader } from './utils/excel-reader';
+import { Config, SourceOrganization } from './types';
+import { TargetOrganizationModel } from './models/strapi-models';
+import { migrationMapping } from './migration/organization-mapping';
+
+// Configuration
+const CONFIG: Config = {
+  STRAPI_URL: process.env.STRAPI_URL || 'http://localhost:1337',
+  STRAPI_TOKEN: process.env.STRAPI_TOKEN || '',
+  EXCEL_FILE: process.env.EXCEL_FILE || '教育公益开放式数据库.xlsx',
+  SHEET_NAME: process.env.SHEET_NAME || null,
+  BATCH_SIZE: parseInt(process.env.BATCH_SIZE || '10'),
+  BATCH_DELAY: parseInt(process.env.BATCH_DELAY || '0'),
+  DRY_RUN: process.env.DRY_RUN === 'true',
+  MAX_ROWS: parseInt(process.env.MAX_ROWS || '0'),
+};
+
+// Data source generator function
+async function* loadOrganizationData(): AsyncGenerator<SourceOrganization> {
+  console.log(`正在读取 Excel 文件: ${CONFIG.EXCEL_FILE}`);
+
+  if (!fs.existsSync(CONFIG.EXCEL_FILE)) {
+    throw new Error(`Excel 文件不存在: ${CONFIG.EXCEL_FILE}`);
+  }
+
+  // Use existing Excel reader
+  const rawOrganizations = ExcelReader.readExcelFile(
+    CONFIG.EXCEL_FILE,
+    CONFIG.SHEET_NAME,
+  );
+
+  if (CONFIG.MAX_ROWS > 0) {
+    rawOrganizations.splice(CONFIG.MAX_ROWS);
+  }
+
+  console.log(`从 Excel 读取到 ${rawOrganizations.length} 条记录`);
+
+  // Yield each organization from existing reader
+  yield* rawOrganizations;
+}
+
+// Main function
+async function main(): Promise<void> {
+  try {
+    console.log(`
+=== Strapi 数据导入工具 ===
+`);
+    if (CONFIG.DRY_RUN)
+      console.log(`
+🔥 DRY RUN 模式 - 不会实际创建数据
+`);
+    const migrator = new RestMigrator(
+      loadOrganizationData,
+      TargetOrganizationModel,
+      migrationMapping,
+      new ConsoleLogger(),
+    );
+
+    console.log('开始数据迁移...\n');
+
+    let count = 0;
+    for await (const organization of migrator.boot()) {
+      count++;
+      console.log(
+        `✅ 成功导入第 ${count} 个组织: ${organization.name || 'Unknown'}`,
+      );
+    }
+
+    console.log(`\n导入完成！共处理 ${count} 个组织`);
+  } catch (error: any) {
+    console.error('导入失败:', error.message);
+    if (error.stack) {
+      console.error('错误堆栈:', error.stack);
+    }
+    process.exit(1);
+  }
+}
+
+// Handle command line arguments
+function parseArgs(): void {
+  const args = process.argv.slice(2);
+
+  if (args.includes('--help') || args.includes('-h')) {
+    console.log(`
+Strapi 数据导入工具 (重构版)
+
+用法:
+  tsx scripts/import-data-refactored.ts [选项]
+
+选项:
+  --dry-run, -d     仅模拟导入，不实际创建数据
+  --help, -h        显示帮助信息
+
+环境变量:
+  STRAPI_URL        Strapi 服务器地址 (默认: http://localhost:1337)
+  STRAPI_TOKEN      Strapi API Token
+  EXCEL_FILE        Excel 文件路径 (默认: 教育公益开放式数据库.xlsx)
+  SHEET_NAME        工作表名称 (默认: 使用第一个工作表)
+  MAX_ROWS          最大处理行数 (默认: 0，表示全部)
+  DRY_RUN           模拟运行 (true/false, 默认: false)
+`);
+    process.exit(0);
+  }
+
+  if (args.includes('--dry-run') || args.includes('-d')) {
+    CONFIG.DRY_RUN = true;
+  }
+}
+
+// Entry point
+if (require.main === module) {
+  parseArgs();
+  main();
+}
+
+export { loadOrganizationData, migrationMapping };

scripts/migration/organization-mapping.ts

@@ -0,0 +1,106 @@
+/**
+ * Migration mapping for MobX-RESTful-migrator
+ */
+
+import { MigrationSchema } from 'mobx-restful-migrator';
+import {
+  SourceOrganization,
+  TargetOrganization,
+  ExtendedUserData,
+} from '../types';
+
+import { AddressTransformer } from '../transformers/address-transformer';
+import { DateTransformer } from '../transformers/date-transformer';
+import { ServiceTransformer } from '../transformers/service-transformer';
+import { QualificationTransformer } from '../transformers/qualification-transformer';
+import { UserTransformer } from '../transformers/user-transformer';
+import { DataUtils } from '../utils/data-utils';
+
+import { TargetUserModel } from '../models/strapi-models';
+
+export const migrationMapping: MigrationSchema<
+  SourceOrganization,
+  TargetOrganization
+> = {
+  常用名称: ({ 常用名称: value }) => ({
+    name: { value: value || '', unique: true },
+  }),
+
+  机构信用代码: ({ 机构信用代码: value }) => ({
+    code: { value: value || '' },
+  }),
+
+  实体类型: ({ 实体类型: value }) => ({
+    entityType: { value: DataUtils.transformEntityType(value) },
+  }),
+
+  注册国籍: ({ 注册国籍: value }) => ({
+    registrationCountry: { value: DataUtils.transformRegistrationCountry(value) },
+  }),
+
+  成立时间: ({ 成立时间: value }) => ({
+    establishedDate: { value: DateTransformer.parseDate(value) },
+  }),
+
+  '机构／项目简介': ({ ['机构／项目简介']: value }) => {
+    const desc = value || '';
+    return {
+      description: { value: DataUtils.cleanDescription(desc) },
+      coverageArea: {
+        value: ServiceTransformer.extractCoverageFromDescription(desc),
+      },
+    };
+  },
+
+  '机构／项目全职人数': ({ ['机构／项目全职人数']: value }) => ({
+    staffCount: { value: DataUtils.parseStaffCount(value) },
+  }),
+
+  注册地: (org) => {
+    const addressData = {
+      province: AddressTransformer.extractProvinceFromAddress(
+        org.注册地 || org.具体地址,
+      ),
+      city: AddressTransformer.extractCityFromAddress(
+        org.注册地 || org.具体地址,
+      ),
+      district: AddressTransformer.extractDistrictFromAddress(
+        org.注册地 || org.具体地址,
+      ),
+      street: org.具体地址 || '',
+    };
+
+    return {
+      address: { value: AddressTransformer.transformAddress(addressData) },
+    };
+  },
+
+  机构官网: (org) => ({
+    services: { value: ServiceTransformer.transformServices(org) },
+  }),
+
+  机构微信公众号: (org) => ({
+    internetContact: { value: ServiceTransformer.transformContacts(org) },
+  }),
+
+  登记管理机关: (org) => ({
+    qualifications: {
+      value: QualificationTransformer.transformQualifications(org),
+    },
+  }),
+
+  机构联系人联系人姓名: (org) => {
+    const userData = UserTransformer.transformUser(org);
+
+    if (!userData) {
+      return {};
+    }
+
+    return {
+      contactUser: {
+        value: userData,
+        model: TargetUserModel,
+      },
+    };
+  },
+};

scripts/models/strapi-models.ts

@@ -0,0 +1,43 @@
+import { HTTPClient } from 'koajax';
+import { StrapiListModel, UserModel } from 'mobx-strapi';
+
+import type { ApiOrganizationOrganization, PluginUsersPermissionsUser as User } from '../../types/generated/contentTypes';
+
+const { STRAPI_API_URL, STRAPI_TOKEN } = process.env;
+
+export const strapiClient = new HTTPClient({
+  baseURI: new URL('api/', STRAPI_API_URL) + '',
+  responseType: 'json',
+}).use(({ request }, next) => {
+  request.headers = {
+    Authorization: `Bearer ${STRAPI_TOKEN}`,
+    ...request.headers,
+    'Strapi-Response-Format': 'v4',
+  };
+  return next();
+});
+
+// Organization model
+export class TargetOrganizationModel extends StrapiListModel<ApiOrganizationOrganization> {
+  baseURI = 'organizations';
+  client = strapiClient;
+}
+
+// User model
+export class TargetUserModel extends UserModel {
+  baseURI = 'users';
+  client = strapiClient;
+
+  override async updateOne(data: Partial<User>, id?: number) {
+    const userData = {
+      ...data,
+      role: data.role || 1,
+      password:
+        data.password || (id ? undefined : this.generateRandomPassword()),
+      confirmed: data.confirmed ?? true,
+    };
+    return super.updateOne(userData, id);
+  }
+
+  private generateRandomPassword = () => Math.random().toString(36).slice(-12);
+}