diff --git a/.env.docker-compose.dev b/.env.docker-compose.dev new file mode 100644 index 000000000..2a3d834c4 --- /dev/null +++ b/.env.docker-compose.dev @@ -0,0 +1,13 @@ +MINIO_ROOT_USER=pubpub-minio-admin +MINIO_ROOT_PASSWORD=pubpub-minio-admin + +ASSETS_BUCKET_NAME=assets.v7.pubpub.org +ASSETS_UPLOAD_KEY=pubpubuser +ASSETS_UPLOAD_SECRET_KEY=pubpubpass +ASSETS_REGION=us-east-1 + +POSTGRES_PORT=54322 +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres + diff --git a/.env.docker-compose.test b/.env.docker-compose.test new file mode 100644 index 000000000..5fc11cef4 --- /dev/null +++ b/.env.docker-compose.test @@ -0,0 +1,40 @@ +MINIO_ROOT_USER=pubpub-minio-admin +MINIO_ROOT_PASSWORD=pubpub-minio-admin + +ASSETS_BUCKET_NAME=byron.v7.pubpub.org +ASSETS_UPLOAD_KEY=pubpubuserrr +ASSETS_UPLOAD_SECRET_KEY=pubpubpass +ASSETS_REGION=us-east-1 +ASSETS_STORAGE_ENDPOINT=http://localhost:9000 + +POSTGRES_PORT=54323 +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres +POSTGRES_HOST=db + +# annoying duplication because jobs uses this version +PGHOST=db +PGPORT=5432 +PGUSER=postgres +PGPASSWORD=postgres +PGDATABASE=postgres + +# this needs to be db:5432 bc that's what it is in the app-network +# if you are running this from outside the docker network, you need to use +# @localhost:${POSTGRES_PORT} instead +DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres + + +JWT_SECRET=xxx +MAILGUN_SMTP_PASSWORD=xxx +GCLOUD_KEY_FILE=xxx + +MAILGUN_SMTP_HOST=inbucket +MAILGUN_SMTP_PORT=2500 +# this needs to be localhost:54324 instead of inbucket:9000 bc we are almost always running the integration tests from outside the docker network +INBUCKET_URL=http://localhost:54324 +MAILGUN_SMTP_USERNAME=omitted +OTEL_SERVICE_NAME=core.core +PUBPUB_URL=http://localhost:3000 +API_KEY=xxx diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..435c7f489 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://json.schemastore.org/dependabot-2.0.json +# Dependabot configuration file +# See documentation: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + # package.json + pnpm catalog updates + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 10 + labels: + - "dependencies" + - "npm" + commit-message: + prefix: "npm" + include: "scope" + # group all minor and patch updates together + groups: + minor-patch-dependencies: + patterns: + - "*" + update-types: + - "minor" + - "patch" + + # GitHub Actions updates + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "github-actions" + commit-message: + prefix: "github-actions" + include: "scope" + + # docker updates + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 5 + labels: + - "dependencies" + - "docker" + commit-message: + prefix: "docker" + include: "scope" diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 000000000..48bc99a69 --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,80 @@ +name: Build Docs + +on: + workflow_call: + inputs: + preview: + type: boolean + required: true + +jobs: + build-docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + with: + # necessary in order to show latest updates in docs + fetch-depth: 0 + uses: actions/checkout@v4 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.13.1 + + - uses: pnpm/action-setup@v4 + name: Install pnpm + with: + run_install: false + + - name: Get pnpm store directory + id: get-store-path + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT + + - name: Setup pnpm cache + uses: actions/cache@v4 + with: + path: ${{ steps.get-store-path.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + # - name: Cache turbo + # uses: actions/cache@v4 + # with: + # path: .turbo + # key: ${{ runner.os }}-turbo-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-turbo- + + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + + - name: set pr number if preview + id: set-pr-number + if: inputs.preview == true + run: | + echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT + + - name: Build docs + env: + PR_NUMBER: ${{ steps.set-pr-number.outputs.PR_NUMBER }} + run: pnpm --filter docs build + + - name: Deploy docs main 🚀 + if: inputs.preview == false + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: docs/out + branch: gh-pages + clean-exclude: pr-preview + force: false + + - name: Deploy docs preview + if: inputs.preview == true + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: docs/out + action: deploy diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64d487d05..37565ca67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ jobs: runs-on: ubuntu-latest env: COMPOSE_FILE: docker-compose.test.yml + ENV_FILE: .env.docker-compose.test steps: - name: Checkout uses: actions/checkout@v4 @@ -26,7 +27,7 @@ jobs: - name: Install Node.js uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22.13.1 - uses: pnpm/action-setup@v4 name: Install pnpm @@ -55,8 +56,8 @@ jobs: restore-keys: | ${{ runner.os }}-turbo- - - name: Start up DB - run: docker compose --profile test up -d + - name: Start test dependencies + run: pnpm test:setup - name: Install dependencies run: pnpm install --frozen-lockfile --prefer-offline @@ -66,11 +67,6 @@ jobs: - name: Run migrations run: pnpm --filter core migrate-test - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres - - - name: generate prisma - run: pnpm --filter core prisma generate - name: Run prettier run: pnpm format diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8b05f8fbd..4d2e9d5b2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,6 +20,8 @@ jobs: integration-tests: name: Integration tests runs-on: ubuntu-latest + env: + ENV_FILE: .env.docker-compose.test steps: - name: Checkout uses: actions/checkout@v4 @@ -27,7 +29,7 @@ jobs: - name: Install Node.js uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22.13.1 - uses: pnpm/action-setup@v4 name: Install pnpm @@ -48,31 +50,6 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline - - - name: Start up DB - run: docker compose -f docker-compose.test.yml --profile test up -d - - - name: p:build - run: pnpm p:build - - - name: Run migrations - run: pnpm --filter core prisma migrate deploy - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres - - - name: generate prisma - run: pnpm --filter core prisma generate - - - name: seed db - run: pnpm --filter core prisma db seed - env: - # 20241126: this prevents the arcadia seed from running, which contains a ton of pubs which potentially might slow down the tests - MINIMAL_SEED: true - SKIP_VALIDATION: true - DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -100,10 +77,26 @@ jobs: echo "jobs_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-jobs}:$IMAGE_TAG" >> $GITHUB_OUTPUT echo "base_label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX:$IMAGE_TAG" >> $GITHUB_OUTPUT + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + + - name: Start up db images + run: pnpm test:setup + + - name: p:build + run: pnpm p:build + + - name: Run migrations and seed + run: pnpm --filter core db:test:reset + env: + # 20241126: this prevents the arcadia seed from running, which contains a ton of pubs which potentially might slow down the tests + MINIMAL_SEED: true + SKIP_VALIDATION: true + - run: pnpm --filter core exec playwright install chromium --with-deps - - name: Start up core - run: docker compose -f docker-compose.test.yml --profile integration up -d + - name: Start up core etc + run: pnpm integration:setup env: INTEGRATION_TESTS_IMAGE: ${{steps.label.outputs.core_label}} JOBS_IMAGE: ${{steps.label.outputs.jobs_label}} @@ -121,7 +114,7 @@ jobs: INTEGRATION_TEST_HOST: localhost - name: Print container logs - if: failure() + if: ${{failure() || cancelled()}} run: docker compose -f docker-compose.test.yml --profile integration logs - name: Upload playwright snapshots artifact diff --git a/.github/workflows/ecrbuild-all.yml b/.github/workflows/ecrbuild-all.yml index 154f87bca..c7a76f039 100644 --- a/.github/workflows/ecrbuild-all.yml +++ b/.github/workflows/ecrbuild-all.yml @@ -9,6 +9,20 @@ on: required: true AWS_SECRET_ACCESS_KEY: required: true + inputs: + publish_to_ghcr: + type: boolean + default: false + outputs: + core-image: + description: "Core image SHA" + value: ${{ jobs.build-core.outputs.image-sha }} + base-image: + description: "Base image SHA" + value: ${{ jobs.build-base.outputs.image-sha }} + jobs-image: + description: "Jobs image SHA" + value: ${{ jobs.build-jobs.outputs.image-sha }} jobs: emit-sha-tag: @@ -26,6 +40,9 @@ jobs: build-base: uses: ./.github/workflows/ecrbuild-template.yml + with: + publish_to_ghcr: ${{ inputs.publish_to_ghcr }} + ghcr_image_name: platform-migrations secrets: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -36,6 +53,8 @@ jobs: # - build-base with: package: core + publish_to_ghcr: ${{ inputs.publish_to_ghcr }} + ghcr_image_name: platform # we require a bigger lad # We are now public, default public runner is big enough # runner: ubuntu-latest-m @@ -50,6 +69,8 @@ jobs: with: package: jobs target: jobs + publish_to_ghcr: ${{ inputs.publish_to_ghcr }} + ghcr_image_name: platform-jobs secrets: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/ecrbuild-template.yml b/.github/workflows/ecrbuild-template.yml index 89b2c30b5..5aa8934d1 100644 --- a/.github/workflows/ecrbuild-template.yml +++ b/.github/workflows/ecrbuild-template.yml @@ -12,6 +12,16 @@ on: default: ubuntu-latest target: type: string + publish_to_ghcr: + type: boolean + default: false + ghcr_image_name: + type: string + required: false + outputs: + image-sha: + description: "Image SHA" + value: ${{ jobs.build.outputs.image-sha }} secrets: AWS_ACCESS_KEY_ID: required: true @@ -28,6 +38,8 @@ jobs: build: name: Build runs-on: ${{ inputs.runner }} + outputs: + image-sha: ${{ steps.label.outputs.label }} steps: - name: Checkout @@ -45,6 +57,13 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # necessary in order to upload build source maps to sentry - name: Get sentry token id: sentry-token @@ -75,6 +94,16 @@ jobs: echo "target=${TARGET:-next-app-${PACKAGE}}" >> $GITHUB_OUTPUT fi echo "label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX$package_suffix:$sha_short" >> $GITHUB_OUTPUT + if [[ ${{ inputs.publish_to_ghcr }} == "true" && -n ${{ inputs.ghcr_image_name }} ]] + then + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + + echo "ghcr_latest_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:latest" >> $GITHUB_OUTPUT + + echo "ghcr_sha_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:$sha_short" >> $GITHUB_OUTPUT + + echo "ghcr_timestamp_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:$TIMESTAMP" >> $GITHUB_OUTPUT + fi - name: Check if SENTRY_AUTH_TOKEN is set run: | @@ -85,7 +114,7 @@ jobs: fi - name: Build, tag, and push image to Amazon ECR - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 id: build-image env: REGISTRY_REF: ${{steps.login-ecr.outputs.registry}}/${{env.ECR_REPOSITORY_PREFIX}}-${{env.PACKAGE}}:cache @@ -103,6 +132,10 @@ jobs: secrets: | SENTRY_AUTH_TOKEN=${{ env.SENTRY_AUTH_TOKEN }} target: ${{ steps.label.outputs.target }} - tags: ${{ steps.label.outputs.label }} + tags: | + ${{ steps.label.outputs.label }} + ${{ steps.label.outputs.ghcr_latest_label }} + ${{ steps.label.outputs.ghcr_sha_label }} + ${{ steps.label.outputs.ghcr_timestamp_label }} platforms: linux/amd64 push: true diff --git a/.github/workflows/on_main.yml b/.github/workflows/on_main.yml index 99b0bcb9b..976083570 100644 --- a/.github/workflows/on_main.yml +++ b/.github/workflows/on_main.yml @@ -14,6 +14,8 @@ jobs: build-all: needs: ci uses: ./.github/workflows/ecrbuild-all.yml + with: + publish_to_ghcr: true secrets: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -38,3 +40,12 @@ jobs: secrets: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + + deploy-docs: + permissions: + contents: write + pages: write + pull-requests: write + uses: ./.github/workflows/build-docs.yml + with: + preview: false diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 0b4176a88..edffc7fd1 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -4,22 +4,43 @@ name: PR Updated triggers on: pull_request: - types: - - opened - - synchronize + types: [labeled, unlabeled, synchronize, closed, reopened, opened] env: AWS_REGION: us-east-1 +permissions: + id-token: write + contents: read + jobs: + path-filter: + runs-on: ubuntu-latest + if: github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || github.event.action == 'closed' + outputs: + docs: ${{ steps.changes.outputs.docs }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + docs: + - 'docs/**' + ci: + if: github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || (github.event.action == 'labeled' && github.event.label.name == 'preview') uses: ./.github/workflows/ci.yml + build-all: + if: github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || (github.event.action == 'labeled' && github.event.label.name == 'preview') uses: ./.github/workflows/ecrbuild-all.yml secrets: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + e2e: + if: github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' needs: - build-all # could theoretically be skipped, but in practice is always faster @@ -29,3 +50,72 @@ jobs: secrets: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + + deploy-preview: + uses: ./.github/workflows/pull-preview.yml + needs: + - build-all + permissions: + contents: read + deployments: write + pull-requests: write + statuses: write + with: + PLATFORM_IMAGE: ${{ needs.build-all.outputs.core-image }} + JOBS_IMAGE: ${{ needs.build-all.outputs.jobs-image }} + MIGRATIONS_IMAGE: ${{ needs.build-all.outputs.base-image }} + PUBLIC_URL: ${{ github.event.repository.html_url }} + AWS_REGION: "us-east-1" + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GH_PAT_PR_PREVIEW_CLEANUP: ${{ secrets.GH_PAT_PR_PREVIEW_CLEANUP }} + + close-preview: + uses: ./.github/workflows/pull-preview.yml + if: github.event.action == 'closed' || (github.event.action == 'unlabeled' && github.event.label.name == 'preview') + permissions: + contents: read + deployments: write + pull-requests: write + statuses: write + with: + PLATFORM_IMAGE: "x" # not used + JOBS_IMAGE: "x" # not used + MIGRATIONS_IMAGE: "x" # not used + PUBLIC_URL: ${{ github.event.repository.html_url }} + AWS_REGION: "us-east-1" + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GH_PAT_PR_PREVIEW_CLEANUP: ${{ secrets.GH_PAT_PR_PREVIEW_CLEANUP }} + + deploy-docs-preview: + permissions: + contents: write + pages: write + pull-requests: write + needs: + - path-filter + if: (github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize') && needs.path-filter.outputs.docs == 'true' + uses: ./.github/workflows/build-docs.yml + with: + preview: true + + close-docs-preview: + needs: + - path-filter + permissions: + contents: write + pages: write + pull-requests: write + if: github.event.action == 'closed' && needs.path-filter.outputs.docs == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Close docs preview + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: docs/out + action: remove diff --git a/.github/workflows/pull-preview.yml b/.github/workflows/pull-preview.yml new file mode 100644 index 000000000..65b11f4db --- /dev/null +++ b/.github/workflows/pull-preview.yml @@ -0,0 +1,77 @@ +on: + workflow_call: + inputs: + PLATFORM_IMAGE: + required: true + type: string + JOBS_IMAGE: + required: true + type: string + MIGRATIONS_IMAGE: + required: true + type: string + PUBLIC_URL: + required: true + type: string + AWS_REGION: + required: true + type: string + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + GH_PAT_PR_PREVIEW_CLEANUP: + required: true + +permissions: + contents: read + deployments: write + pull-requests: write + statuses: write + +jobs: + preview: + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Copy .env file + run: cp ./self-host/.env.example ./self-host/.env + + - name: Configure pullpreview + env: + PLATFORM_IMAGE: ${{ inputs.PLATFORM_IMAGE }} + JOBS_IMAGE: ${{ inputs.JOBS_IMAGE }} + MIGRATIONS_IMAGE: ${{ inputs.MIGRATIONS_IMAGE }} + run: | + sed -i "s|image: PLATFORM_IMAGE|image: $PLATFORM_IMAGE|" docker-compose.preview.yml + sed -i "s|image: JOBS_IMAGE|image: $JOBS_IMAGE|" docker-compose.preview.yml + sed -i "s|image: MIGRATIONS_IMAGE|image: $MIGRATIONS_IMAGE|" docker-compose.preview.yml + sed -i "s|email someone@example.com|email dev@pubpub.org|" self-host/caddy/Caddyfile + sed -i "s|example.com|{\$PUBLIC_URL}|" self-host/caddy/Caddyfile + + - name: Get ECR token + id: ecrtoken + run: echo "value=$(aws ecr get-login-password --region us-east-1)" >> $GITHUB_OUTPUT + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "us-east-1" + + - uses: pullpreview/action@v5 + with: + label: preview + admins: 3mcd + compose_files: ./self-host/docker-compose.yml,docker-compose.preview.yml + default_port: 443 + instance_type: small + ports: 80,443,9001 + registries: docker://AWS:${{steps.ecrtoken.outputs.value}}@246372085946.dkr.ecr.us-east-1.amazonaws.com + github_token: ${{ secrets.GH_PAT_PR_PREVIEW_CLEANUP }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ inputs.AWS_REGION }} + PULLPREVIEW_LOGGER_LEVEL: DEBUG diff --git a/.gitignore b/.gitignore index 288bdc3ea..ccc4f294c 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,6 @@ core/supabase/.temp *storybook.log storybook-static -./playwright \ No newline at end of file +./playwright + +.local_data diff --git a/.nvmrc b/.nvmrc index 9aef5aab6..437793775 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v20.17.0 \ No newline at end of file +v22.13.1 \ No newline at end of file diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 7636918d8..e102df0a1 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -3,6 +3,8 @@ "ms-playwright.playwright", "YoavBls.pretty-ts-errors", "esbenp.prettier-vscode", - "dbaeumer.vscode-eslint" + "dbaeumer.vscode-eslint", + // for yaml autocompletion using the # yaml-language-server: $schema=... directive + "redhat.vscode-yaml" ] } diff --git a/Dockerfile b/Dockerfile index ee68193b0..f23946bb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ # If you need more help, visit the Dockerfile reference guide at # https://docs.docker.com/go/dockerfile-reference/ -ARG NODE_VERSION=20.17.0 +ARG NODE_VERSION=22.13.1 +ARG ALPINE_VERSION=3.20 ARG PACKAGE ARG PORT=3000 @@ -14,7 +15,7 @@ ARG PNPM_VERSION=9.10.0 ################################################################################ # Use node image for base image for all stages. -FROM node:${NODE_VERSION}-alpine as base +FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} as base # these are necessary to be able to use them inside of `base` ARG BASE_IMAGE diff --git a/README.md b/README.md index 4eb2c7ba2..bf6bf5230 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,11 @@ root - `jobs` holds the job queueing and scheduling service used by `core`. - `packages` holds libraries and npm packages that are shared by `core`, `jobs`, and `infrastructure`. -To avoid inconsistencies and difficult-to-track errors, we specify a particular version of node in `/.nvmrc` (currently `v20.17.0`). We recommend using [nvm](https://github.com/nvm-sh/nvm) to ensure you're using the same version. +To avoid inconsistencies and difficult-to-track errors, we specify a particular version of node in `/.nvmrc` (currently `v22.13.1`). We recommend using [nvm](https://github.com/nvm-sh/nvm) to ensure you're using the same version. ## Local Installation -This package runs the version of node specified in `.nvmrc` (currently `v20.17.0`) and uses pnpm for package management. All following commands are run from the root of this package. +This package runs the version of node specified in `.nvmrc` (currently `v22.13.1`) and uses pnpm for package management. All following commands are run from the root of this package. To get started, clone the repository and install the version of node specified in `.nvmrc` (we recommend using [nvm](https://github.com/nvm-sh/nvm). @@ -52,7 +52,7 @@ pnpm install pnpm build ``` -**Running build when getting started with this repo is important to make sure the any prebuild scripts run (e.g. Prisma generate).** +**Running build when getting started with this repo is important to make sure the any prebuild scripts run** Depending on which app or package you are doing work on, you may need to create a .env.local file. See each package's individual README.md file for further details. diff --git a/config/prettier/index.js b/config/prettier/index.js index dca76965a..310c789ae 100644 --- a/config/prettier/index.js +++ b/config/prettier/index.js @@ -18,7 +18,7 @@ const config = { // "prettier-plugin-jsdoc", ], // tailwindConfig: fileURLToPath( - // new URL("../../tooling/tailwind/web.ts", import.meta.url), + // new URL("../../packages/ui/tailwind.config.cjs", import.meta.url) // ), tailwindFunctions: ["cn", "cva"], importOrder: [ diff --git a/core/.env.development b/core/.env.development index eee181691..1ef1fadb7 100644 --- a/core/.env.development +++ b/core/.env.development @@ -6,8 +6,11 @@ PUBPUB_URL="http://localhost:3000" ASSETS_BUCKET_NAME="assets.v7.pubpub.org" ASSETS_REGION="us-east-1" -ASSETS_UPLOAD_KEY="xxx" -ASSETS_UPLOAD_SECRET_KEY="xxx" +# mninio defaults +ASSETS_UPLOAD_KEY="pubpubuser" +ASSETS_UPLOAD_SECRET_KEY="pubpubpass" +ASSETS_STORAGE_ENDPOINT="http://localhost:9000" + MAILGUN_SMTP_PASSWORD="xxx" MAILGUN_SMTP_USERNAME="xxx" diff --git a/core/.env.test b/core/.env.test index 9f486764f..a030cfd46 100644 --- a/core/.env.test +++ b/core/.env.test @@ -1,13 +1,15 @@ -DATABASE_URL=postgresql://postgres:postgres@localhost:5433/postgres +DATABASE_URL=postgresql://postgres:postgres@localhost:54323/postgres PUBPUB_URL=http://localhost:3000 MAILGUN_SMTP_HOST=localhost MAILGUN_SMTP_PORT=54325 API_KEY="super_secret_key" -ASSETS_BUCKET_NAME="assets.v7.pubpub.org" -ASSETS_REGION="us-east-1" -ASSETS_UPLOAD_KEY="xxx" -ASSETS_UPLOAD_SECRET_KEY="xxx" +ASSETS_BUCKET_NAME=byron.v7.pubpub.org +ASSETS_UPLOAD_KEY=pubpubuserrr +ASSETS_UPLOAD_SECRET_KEY=pubpubpass +ASSETS_REGION=us-east-1 +ASSETS_STORAGE_ENDPOINT="http://localhost:9000" + MAILGUN_SMTP_PASSWORD="xxx" MAILGUN_SMTP_USERNAME="xxx" @@ -17,3 +19,4 @@ HONEYCOMB_API_KEY="xxx" KYSELY_DEBUG="true" GCLOUD_KEY_FILE='xxx' + diff --git a/core/.github/workflows/playwright.yml b/core/.github/workflows/playwright.yml index f314305dc..b050f147e 100644 --- a/core/.github/workflows/playwright.yml +++ b/core/.github/workflows/playwright.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: lts/* + node-version: 22.13.1 - name: Install dependencies run: npm install -g pnpm && pnpm install - name: Install Playwright Browsers diff --git a/core/README.md b/core/README.md index 1b62053e5..58a5cea34 100644 --- a/core/README.md +++ b/core/README.md @@ -38,6 +38,8 @@ pnpm run dev ## Prisma +We currently only use Prisma for managing migrations. + The Prisma [Quickstart guide](https://www.prisma.io/docs/getting-started/quickstart) is how our prisma folder was initially created. That set of instructions has useful pointers for doing things like db migrations. The `~/prisma/seed.ts` file will initiate the database with a set of data. This seed is run using `pnpm reset`. You will have to run this each time you stop and start supabase since doing so clears the database. @@ -46,13 +48,13 @@ Explore with `pnpm prisma-studio`. ## Folder structure -- `/actions` Configuration/lib for the action framework. -- `/app` The Next.JS [app directory](https://nextjs.org/docs/app/building-your-application/routing). -- `/lib` Functions that are re-used in multiple locations throughout the codebase. Akin to a `/utils` folder. -- `/prisma` Config and functions for using Prisma -- `/kysely` Config and functions for using Kysely -- `/public` Static files that will be publicly available at `https://[URL]/`. -- `/playwright` End-to-end tests +- `/actions` Configuration/lib for the action framework. +- `/app` The Next.JS [app directory](https://nextjs.org/docs/app/building-your-application/routing). +- `/lib` Functions that are re-used in multiple locations throughout the codebase. Akin to a `/utils` folder. +- `/prisma` Config and functions for using Prisma +- `/kysely` Config and functions for using Kysely +- `/public` Static files that will be publicly available at `https://[URL]/`. +- `/playwright` End-to-end tests ## Authentication diff --git a/core/actions/_lib/runActionInstance.db.test.ts b/core/actions/_lib/runActionInstance.db.test.ts index 7e0914716..3d00083ea 100644 --- a/core/actions/_lib/runActionInstance.db.test.ts +++ b/core/actions/_lib/runActionInstance.db.test.ts @@ -10,7 +10,7 @@ const { getTrx, rollback, commit } = createForEachMockedTransaction(); const pubTriggerTestSeed = async () => { const slugName = `test-server-pub-${new Date().toISOString()}`; - const { createSeed } = await import("~/prisma/seed/seedCommunity"); + const { createSeed } = await import("~/prisma/seed/createSeed"); return createSeed({ community: { diff --git a/core/actions/_lib/runActionInstance.ts b/core/actions/_lib/runActionInstance.ts index b4a6ea0b9..f0906716f 100644 --- a/core/actions/_lib/runActionInstance.ts +++ b/core/actions/_lib/runActionInstance.ts @@ -18,7 +18,7 @@ import type { ClientException, ClientExceptionOptions } from "~/lib/serverAction import { db } from "~/kysely/database"; import { hydratePubValues } from "~/lib/fields/utils"; import { createLastModifiedBy } from "~/lib/lastModifiedBy"; -import { getPubsWithRelatedValuesAndChildren } from "~/lib/server"; +import { getPubsWithRelatedValues } from "~/lib/server"; import { autoRevalidate } from "~/lib/server/cache/autoRevalidate"; import { isClientException } from "~/lib/serverActions"; import { getActionByName } from "../api"; @@ -40,7 +40,7 @@ const _runActionInstance = async ( ): Promise => { const isActionUserInitiated = "userId" in args; - const pubPromise = getPubsWithRelatedValuesAndChildren( + const pubPromise = getPubsWithRelatedValues( { pubId: args.pubId, communityId: args.communityId, diff --git a/core/actions/datacite/action.ts b/core/actions/datacite/action.ts index 534a07131..65d84821c 100644 --- a/core/actions/datacite/action.ts +++ b/core/actions/datacite/action.ts @@ -9,13 +9,13 @@ export const action = defineAction({ name: Action.datacite, config: { schema: z.object({ - doi: z.string(), + doi: z.string().optional(), doiPrefix: z.string().optional(), - doiSuffix: z.string(), + doiSuffix: z.string().optional(), title: z.string(), url: z.string(), publisher: z.string(), - publicationDate: z.date(), + publicationDate: z.coerce.date(), creator: z.string(), creatorName: z.string(), }), @@ -48,13 +48,13 @@ export const action = defineAction({ }, params: { schema: z.object({ - doi: z.string(), + doi: z.string().optional(), doiPrefix: z.string().optional(), - doiSuffix: z.string(), + doiSuffix: z.string().optional(), title: z.string(), url: z.string(), publisher: z.string(), - publicationDate: z.date(), + publicationDate: z.coerce.date(), creator: z.string(), creatorName: z.string(), }), diff --git a/core/actions/datacite/run.test.ts b/core/actions/datacite/run.test.ts index 2870ae09f..1b3e44951 100644 --- a/core/actions/datacite/run.test.ts +++ b/core/actions/datacite/run.test.ts @@ -32,7 +32,7 @@ vitest.mock("~/lib/env/env.mjs", () => { vitest.mock("~/lib/server", () => { return { - getPubsWithRelatedValuesAndChildren: () => { + getPubsWithRelatedValues: () => { return { ...pub, values: [] }; }, updatePub: vitest.fn(() => { @@ -110,7 +110,6 @@ const pub = { relatedPubId: null, }, ], - children: [], communityId: "" as CommunitiesId, createdAt: new Date(), updatedAt: new Date(), diff --git a/core/actions/datacite/run.ts b/core/actions/datacite/run.ts index a098a27b8..da5770bc6 100644 --- a/core/actions/datacite/run.ts +++ b/core/actions/datacite/run.ts @@ -6,11 +6,11 @@ import type { ProcessedPub } from "contracts"; import type { PubsId } from "db/public"; import { assert, AssertionError, expect } from "utils"; -import type { ActionPub, ActionPubType } from "../types"; +import type { ActionPub } from "../types"; import type { action } from "./action"; import type { components } from "./types"; import { env } from "~/lib/env/env.mjs"; -import { getPubsWithRelatedValuesAndChildren, updatePub } from "~/lib/server"; +import { getPubsWithRelatedValues, updatePub } from "~/lib/server"; import { isClientExceptionOptions } from "~/lib/serverActions"; import { defineRun } from "../types"; @@ -18,9 +18,7 @@ type ConfigSchema = z.infer<(typeof action)["config"]["schema"]>; type Config = ConfigSchema & { pubFields: { [K in keyof ConfigSchema]?: string[] } }; type Payload = components["schemas"]["Doi"]; -type RelatedPubs = Awaited< - ReturnType> ->[number]["values"]; +type RelatedPubs = Awaited>>[number]["values"]; const encodeDataciteCredentials = (username: string, password: string) => Buffer.from(`${username}:${password}`).toString("base64"); @@ -65,7 +63,7 @@ const makeDatacitePayload = async (pub: ActionPub, config: Config): Promise Boolean(recipientMember), + type: DependencyType.DISABLES, + }, + ], }, description: "Send an email to one or more users", params: { schema: z .object({ - recipient: z + recipientEmail: z.string().email().describe("Recipient email address").optional(), + recipientMember: z .string() .uuid() .describe( - "Recipient|Overrides the recipient user specified in the action config." + "Recipient Member|Overrides the recipient community member specified in the action config." ) .optional(), subject: stringWithTokens() @@ -46,10 +60,21 @@ export const action = defineAction({ }) .optional(), fieldConfig: { - recipient: { + recipientEmail: { + allowedSchemas: true, + }, + recipientMember: { fieldType: "custom", }, }, + dependencies: [ + { + sourceField: "recipientMember", + targetField: "recipientEmail", + when: (recipientMember) => Boolean(recipientMember), + type: DependencyType.DISABLES, + }, + ], }, icon: Mail, tokens: { diff --git a/core/actions/email/config/recipient.field.tsx b/core/actions/email/config/recipientMember.field.tsx similarity index 58% rename from core/actions/email/config/recipient.field.tsx rename to core/actions/email/config/recipientMember.field.tsx index 7b7f33d26..4b1187d3a 100644 --- a/core/actions/email/config/recipient.field.tsx +++ b/core/actions/email/config/recipientMember.field.tsx @@ -1,7 +1,7 @@ import type { CommunityMembershipsId } from "db/public"; import { defineActionFormFieldServerComponent } from "~/actions/_lib/custom-form-field/defineConfigServerComponent"; -import { MemberSelectServer } from "~/app/components/MemberSelect/MemberSelectServer"; +import { MemberSelectClientFetch } from "~/app/components/MemberSelect/MemberSelectClientFetch"; import { db } from "~/kysely/database"; import { autoCache } from "~/lib/server/cache/autoCache"; import { action } from "../action"; @@ -14,17 +14,12 @@ const component = defineActionFormFieldServerComponent( db.selectFrom("communities").selectAll().where("id", "=", communityId) ).executeTakeFirstOrThrow(); - const queryParamName = `recipient-${actionInstance.id?.split("-").pop()}`; - const query = pageContext.searchParams?.[queryParamName] as string | undefined; - return ( - ); } diff --git a/core/actions/email/params/recipient.field.tsx b/core/actions/email/params/recipientMember.field.tsx similarity index 58% rename from core/actions/email/params/recipient.field.tsx rename to core/actions/email/params/recipientMember.field.tsx index 945199dcc..6473817e9 100644 --- a/core/actions/email/params/recipient.field.tsx +++ b/core/actions/email/params/recipientMember.field.tsx @@ -1,7 +1,7 @@ import type { CommunityMembershipsId } from "db/public"; import { defineActionFormFieldServerComponent } from "~/actions/_lib/custom-form-field/defineConfigServerComponent"; -import { MemberSelectServer } from "~/app/components/MemberSelect/MemberSelectServer"; +import { MemberSelectClientFetch } from "~/app/components/MemberSelect/MemberSelectClientFetch"; import { db } from "~/kysely/database"; import { autoCache } from "~/lib/server/cache/autoCache"; import { action } from "../action"; @@ -14,17 +14,12 @@ const component = defineActionFormFieldServerComponent( db.selectFrom("communities").selectAll().where("id", "=", communityId) ).executeTakeFirstOrThrow(); - const queryParamName = `recipient-${actionInstance.id?.split("-").pop()}`; - const query = pageContext.searchParams?.[queryParamName] as string | undefined; - return ( - ); } diff --git a/core/actions/email/run.ts b/core/actions/email/run.ts index f9ffd9a1f..7818ee85f 100644 --- a/core/actions/email/run.ts +++ b/core/actions/email/run.ts @@ -4,7 +4,7 @@ import { jsonObjectFrom } from "kysely/helpers/postgres"; import type { CommunityMembershipsId } from "db/public"; import { logger } from "logger"; -import { expect } from "utils"; +import { assert, expect } from "utils"; import type { action } from "./action"; import type { @@ -12,10 +12,11 @@ import type { RenderWithPubPub, } from "~/lib/server/render/pub/renderWithPubUtils"; import { db } from "~/kysely/database"; -import { getPubsWithRelatedValuesAndChildren } from "~/lib/server"; +import { getPubsWithRelatedValues } from "~/lib/server"; import { getCommunitySlug } from "~/lib/server/cache/getCommunitySlug"; import * as Email from "~/lib/server/email"; import { renderMarkdownWithPub } from "~/lib/server/render/pub/renderMarkdownWithPub"; +import { isClientException } from "~/lib/serverActions"; import { defineRun } from "../types"; export const run = defineRun(async ({ pub, config, args, communityId }) => { @@ -29,7 +30,7 @@ export const run = defineRun(async ({ pub, config, args, communit // will redundantly load the child pub. Ideally we would lazily fetch and // cache the parent pub while processing the email template. if (parentId) { - parentPub = await getPubsWithRelatedValuesAndChildren( + parentPub = await getPubsWithRelatedValues( { pubId: parentId, communityId }, { withPubType: true, @@ -38,28 +39,37 @@ export const run = defineRun(async ({ pub, config, args, communit ); } - const recipientId = expect(args?.recipient ?? config.recipient) as CommunityMembershipsId; - - // TODO: similar to the assignee, the recipient args/config should accept - // the pub assignee, a pub field, a static email address, a member, or a - // member group. - const recipient = await db - .selectFrom("community_memberships") - .select((eb) => [ - "community_memberships.id", - jsonObjectFrom( - eb - .selectFrom("users") - .whereRef("users.id", "=", "community_memberships.userId") - .selectAll("users") - ) - .$notNull() - .as("user"), - ]) - .where("id", "=", recipientId) - .executeTakeFirstOrThrow( - () => new Error(`Could not find member with ID ${recipientId}`) - ); + const recipientEmail = args?.recipientEmail ?? config.recipientEmail; + const recipientMemberId = (args?.recipientMember ?? config.recipientMember) as + | CommunityMembershipsId + | undefined; + + assert( + recipientEmail !== undefined || recipientMemberId !== undefined, + "No email recipient was specified" + ); + + let recipient: RenderWithPubContext["recipient"] | undefined; + + if (recipientMemberId !== undefined) { + recipient = await db + .selectFrom("community_memberships") + .select((eb) => [ + "community_memberships.id", + jsonObjectFrom( + eb + .selectFrom("users") + .whereRef("users.id", "=", "community_memberships.userId") + .selectAll("users") + ) + .$notNull() + .as("user"), + ]) + .where("id", "=", recipientMemberId) + .executeTakeFirstOrThrow( + () => new Error(`Could not find member with ID ${recipientMemberId}`) + ); + } const renderMarkdownWithPubContext = { communityId, @@ -79,13 +89,34 @@ export const run = defineRun(async ({ pub, config, args, communit true ); - await Email.generic({ - to: recipient.user.email, + const result = await Email.generic({ + to: expect(recipient?.user.email ?? recipientEmail), subject, html, }).send(); + + if (isClientException(result)) { + logger.error({ + msg: "An error occurred while sending an email", + error: result.error, + pub, + config, + args, + renderMarkdownWithPubContext, + }); + } else { + logger.info({ + msg: "Successfully sent email", + pub, + config, + args, + renderMarkdownWithPubContext, + }); + } + + return result; } catch (error) { - logger.error({ msg: "email", error }); + logger.error({ msg: "Failed to send email", error }); return { title: "Failed to Send Email", @@ -93,12 +124,4 @@ export const run = defineRun(async ({ pub, config, args, communit cause: error, }; } - - logger.info({ msg: "email", pub, config, args }); - - return { - success: true, - report: "Email sent", - data: {}, - }; }); diff --git a/core/actions/googleDriveImport/OutputField.tsx b/core/actions/googleDriveImport/OutputField.tsx index 7e03a5118..8a24e4d1b 100644 --- a/core/actions/googleDriveImport/OutputField.tsx +++ b/core/actions/googleDriveImport/OutputField.tsx @@ -1,8 +1,7 @@ "use client"; -import type { CoreSchemaType } from "@prisma/client"; - import type { PubFieldSchemaId, PubFieldsId } from "db/public"; +import { CoreSchemaType } from "db/public"; import { FormControl, FormField, FormItem, FormLabel } from "ui/form"; import { Info } from "ui/icon"; import { usePubFieldContext } from "ui/pubFields"; diff --git a/core/actions/googleDriveImport/config/outputField.field.tsx b/core/actions/googleDriveImport/config/outputField.field.tsx index b22f4ec94..c49a030c3 100644 --- a/core/actions/googleDriveImport/config/outputField.field.tsx +++ b/core/actions/googleDriveImport/config/outputField.field.tsx @@ -1,4 +1,4 @@ -import { CoreSchemaType } from "@prisma/client"; +import { CoreSchemaType } from "db/public"; import { defineActionFormFieldServerComponent } from "../../_lib/custom-form-field/defineConfigServerComponent"; import { action } from "../action"; diff --git a/core/actions/googleDriveImport/discussionSchema.ts b/core/actions/googleDriveImport/discussionSchema.ts new file mode 100644 index 000000000..868377811 --- /dev/null +++ b/core/actions/googleDriveImport/discussionSchema.ts @@ -0,0 +1,446 @@ +import type { DOMOutputSpec, Mark, Node, NodeSpec } from "prosemirror-model"; + +import { Schema } from "prosemirror-model"; + +export const baseNodes: { [key: string]: NodeSpec } = { + doc: { + content: "block+", + attrs: { + meta: { default: {} }, + }, + }, + paragraph: { + selectable: false, + // reactive: true, + content: "inline*", + group: "block", + attrs: { + id: { default: null }, + class: { default: null }, + textAlign: { default: null }, + rtl: { default: null }, + }, + parseDOM: [ + { + tag: "p", + getAttrs: (node) => { + return { + id: (node as Element).getAttribute("id"), + class: (node as Element).getAttribute("class"), + textAlign: (node as Element).getAttribute("data-text-align"), + rtl: (node as Element).getAttribute("data-rtl"), + }; + }, + }, + ], + toDOM: (node) => { + const isEmpty = !node.content || (Array.isArray(node.content) && !node.content.length); + const children = isEmpty ? ["br"] : 0; + return [ + "p", + { + class: node.attrs.class, + ...(node.attrs.id && { id: node.attrs.id }), + ...(node.attrs.textAlign && { "data-text-align": node.attrs.textAlign }), + ...(node.attrs.rtl && { "data-rtl": node.attrs.rtl.toString() }), + }, + children, + ] as DOMOutputSpec; + }, + }, + blockquote: { + content: "block+", + group: "block", + attrs: { + id: { default: null }, + }, + selectable: false, + parseDOM: [ + { + tag: "blockquote", + getAttrs: (node) => { + return { + id: (node as Element).getAttribute("id"), + }; + }, + }, + ], + toDOM: (node) => { + return [ + "blockquote", + { ...(node.attrs.id && { id: node.attrs.id }) }, + 0, + ] as DOMOutputSpec; + }, + }, + horizontal_rule: { + group: "block", + parseDOM: [{ tag: "hr" }], + selectable: true, + toDOM: () => { + return ["div", ["hr"]] as DOMOutputSpec; + }, + }, + heading: { + attrs: { + level: { default: 1 }, + fixedId: { default: "" }, + id: { default: "" }, + textAlign: { default: null }, + rtl: { default: null }, + }, + content: "inline*", + group: "block", + defining: true, + selectable: false, + parseDOM: [1, 2, 3, 4, 5, 6].map((level) => { + return { + tag: `h${level}`, + getAttrs: (node) => { + return { + id: (node as Element).getAttribute("id"), + textAlign: (node as Element).getAttribute("data-text-align"), + rtl: (node as Element).getAttribute("data-rtl"), + level, + }; + }, + }; + }), + toDOM: (node) => { + return [ + `h${node.attrs.level}`, + { + id: node.attrs.fixedId || node.attrs.id, + ...(node.attrs.textAlign && { "data-text-align": node.attrs.textAlign }), + ...(node.attrs.rtl && { "data-rtl": node.attrs.rtl.toString() }), + }, + 0, + ] as DOMOutputSpec; + }, + }, + ordered_list: { + content: "list_item+", + group: "block", + attrs: { + id: { default: null }, + order: { default: 1 }, + rtl: { default: null }, + }, + selectable: false, + parseDOM: [ + { + tag: "ol", + getAttrs: (node) => { + return { + id: (node as Element).getAttribute("id"), + order: (node as Element).hasAttribute("start") + ? +(node as Element).getAttribute("start")! + : 1, + rtl: (node as Element).getAttribute("data-rtl"), + }; + }, + }, + ], + toDOM: (node) => { + return [ + "ol", + { + ...(node.attrs.id && { id: node.attrs.id }), + ...(node.attrs.textAlign && { "data-text-align": node.attrs.textAlign }), + ...(node.attrs.rtl && { "data-rtl": node.attrs.rtl.toString() }), + start: node.attrs.order === 1 ? null : node.attrs.order, + }, + 0, + ] as DOMOutputSpec; + }, + }, + bullet_list: { + content: "list_item+", + group: "block", + attrs: { + id: { default: null }, + rtl: { default: null }, + }, + selectable: false, + parseDOM: [ + { + tag: "ul", + getAttrs: (node) => { + return { + id: (node as Element).getAttribute("id"), + rtl: (node as Element).getAttribute("data-rtl"), + }; + }, + }, + ], + toDOM: (node) => { + return [ + "ul", + { + ...(node.attrs.id && { id: node.attrs.id }), + ...(node.attrs.textAlign && { "data-text-align": node.attrs.textAlign }), + ...(node.attrs.rtl && { "data-rtl": node.attrs.rtl.toString() }), + }, + 0, + ] as DOMOutputSpec; + }, + }, + list_item: { + content: "paragraph block*", + defining: true, + selectable: false, + parseDOM: [{ tag: "li" }], + toDOM: () => { + return ["li", 0] as DOMOutputSpec; + }, + }, + text: { + inline: true, + group: "inline", + toDOM: (node) => { + return node.text!; + }, + }, + hard_break: { + inline: true, + group: "inline", + selectable: false, + parseDOM: [{ tag: "br" }], + toDOM: () => { + return ["br"] as DOMOutputSpec; + }, + }, + image: { + atom: true, + reactive: true, + attrs: { + id: { default: null }, + url: { default: null }, + src: { default: null }, + size: { default: 50 }, // number as percentage + align: { default: "center" }, + caption: { default: "" }, + altText: { default: "" }, + hideLabel: { default: false }, + fullResolution: { default: false }, + href: { default: null }, + }, + parseDOM: [ + { + tag: "figure", + getAttrs: (node) => { + if (node.getAttribute("data-node-type") !== "image") { + return false; + } + return { + id: node.getAttribute("id") || null, + url: node.getAttribute("data-url") || null, + caption: node.getAttribute("data-caption") || "", + size: Number(node.getAttribute("data-size")) || 50, + align: node.getAttribute("data-align") || "center", + altText: node.getAttribute("data-alt-text") || "", + hideLabel: node.getAttribute("data-hide-label") || "", + href: node.getAttribute("data-href") || null, + }; + }, + }, + ], + toDOM: (node) => { + const { url, align, id, altText, caption, size, hideLabel, href } = node.attrs; + return [ + "figure", + { + ...(id && { id }), + "data-node-type": "image", + "data-size": size, + "data-align": align, + "data-url": url, + "data-caption": caption, + "data-href": href, + "data-alt-text": altText, + "data-hide-label": hideLabel, + }, + [ + "img", + { + src: url, + alt: altText || "", + }, + ], + ] as unknown as DOMOutputSpec; + }, + inline: false, + group: "block", + }, + file: { + atom: true, + attrs: { + id: { default: null }, + url: { default: null }, + fileName: { default: null }, + fileSize: { default: null }, + caption: { default: "" }, + }, + parseDOM: [ + { + tag: "figure", + getAttrs: (node) => { + if (node.getAttribute("data-node-type") !== "file") { + return false; + } + return { + id: node.getAttribute("id") || null, + url: node.getAttribute("data-url") || null, + fileName: node.getAttribute("data-file-name") || null, + fileSize: node.getAttribute("data-file-size") || null, + caption: node.getAttribute("data-caption") || "", + }; + }, + }, + ], + toDOM: (node: Node) => { + const attrs = node.attrs; + return [ + "p", + [ + "a", + { + href: attrs.url, + target: "_blank", + rel: "noopener noreferrer", + download: attrs.fileName, + class: `download`, + }, + attrs.fileName, + ], + ]; + }, + inline: false, + group: "block", + }, + code_block: { + content: "text*", + group: "block", + attrs: { + lang: { default: null }, + id: { default: null }, + }, + code: true, + selectable: false, + parseDOM: [ + { + tag: "pre", + getAttrs: (node) => { + return { + id: (node as Element).getAttribute("id"), + }; + }, + preserveWhitespace: "full" as const, + }, + ], + toDOM: (node: Node) => + ["pre", { ...(node.attrs.id && { id: node.attrs.id }) }, ["code", 0]] as DOMOutputSpec, + }, +}; + +export const baseMarks = { + em: { + parseDOM: [ + { tag: "i" }, + { tag: "em" }, + { + style: "font-style", + getAttrs: (value: string) => value === "italic" && null, + }, + ], + toDOM: () => { + return ["em"] as DOMOutputSpec; + }, + }, + + strong: { + parseDOM: [ + { tag: "strong" }, + /* + This works around a Google Docs misbehavior where + pasted content will be inexplicably wrapped in `` + tags with a font-weight normal. + */ + { + tag: "b", + getAttrs: (node: HTMLElement) => node.style.fontWeight !== "normal" && null, + }, + { + style: "font-weight", + getAttrs: (value: string) => /^(bold(er)?|[5-9]\d{2,})$/.test(value) && null, + }, + ], + toDOM: () => { + return ["strong"] as DOMOutputSpec; + }, + }, + link: { + inclusive: false, + attrs: { + href: { default: "" }, + title: { default: null }, + target: { default: null }, + pubEdgeId: { default: null }, + }, + parseDOM: [ + { + tag: "a[href]", + getAttrs: (dom: HTMLElement) => { + if (dom.getAttribute("data-node-type") === "reference") { + return false; + } + return { + href: dom.getAttribute("href"), + title: dom.getAttribute("title"), + target: dom.getAttribute("target"), + pubEdgeId: dom.getAttribute("data-pub-edge-id"), + }; + }, + }, + ], + toDOM: (mark: Mark, inline: boolean) => { + let attrs = mark.attrs; + if (attrs.target && typeof attrs.target !== "string") { + attrs = { ...attrs, target: null }; + } + const { pubEdgeId, ...restAttrs } = attrs; + return ["a", { "data-pub-edge-id": pubEdgeId, ...restAttrs }] as DOMOutputSpec; + }, + }, + sub: { + parseDOM: [{ tag: "sub" }], + toDOM: () => { + return ["sub"] as DOMOutputSpec; + }, + }, + sup: { + parseDOM: [{ tag: "sup" }], + toDOM: () => { + return ["sup"] as DOMOutputSpec; + }, + }, + strike: { + parseDOM: [{ tag: "s" }, { tag: "strike" }, { tag: "del" }], + toDOM: () => { + return ["s"] as DOMOutputSpec; + }, + }, + code: { + parseDOM: [{ tag: "code" }], + toDOM: () => { + return ["code"] as DOMOutputSpec; + }, + }, +}; + +const mySchema = new Schema({ + nodes: baseNodes, + marks: baseMarks, +}); + +export default mySchema; diff --git a/core/actions/googleDriveImport/formatDriveData.ts b/core/actions/googleDriveImport/formatDriveData.ts index 5c56e846f..af06913e4 100644 --- a/core/actions/googleDriveImport/formatDriveData.ts +++ b/core/actions/googleDriveImport/formatDriveData.ts @@ -1,13 +1,26 @@ -import { writeFile } from "fs/promises"; +// import { writeFile } from "fs/promises"; +import type { Root } from "hast"; +import { defaultMarkdownSerializer } from "prosemirror-markdown"; +import { Node } from "prosemirror-model"; import { rehype } from "rehype"; import rehypeFormat from "rehype-format"; +import { visit } from "unist-util-visit"; import type { PubsId } from "db/public"; import type { DriveData } from "./getGDriveFiles"; +import { uploadFileToS3 } from "~/lib/server"; +import schema from "./discussionSchema"; import { + appendFigureAttributes, + cleanUnusedSpans, + formatFigureReferences, + formatLists, + getDescription, processLocalLinks, + removeDescription, + removeEmptyFigCaption, removeGoogleLinkForwards, removeVerboseFormatting, structureAnchors, @@ -23,10 +36,13 @@ import { structureInlineCode, structureInlineMath, structureReferences, + structureTables, structureVideos, } from "./gdocPlugins"; +import { getAssetFile } from "./getGDriveFiles"; export type FormattedDriveData = { + pubDescription: string; pubHtml: string; versions: { [description: `${string}:description`]: string; @@ -35,13 +51,74 @@ export type FormattedDriveData = { }[]; discussions: { id: PubsId; values: {} }[]; }; +const processAssets = async (html: string, pubId: string): Promise => { + const result = await rehype() + .use(() => async (tree: Root) => { + const assetUrls: { [key: string]: string } = {}; + visit(tree, "element", (node: any) => { + const hasSrc = ["img", "video", "audio", "source"].includes(node.tagName); + const isDownload = + node.tagName === "a" && node.properties.className === "file-button"; + if (hasSrc || isDownload) { + const propertyKey = hasSrc ? "src" : "href"; + const originalAssetUrl = node.properties[propertyKey]; + if (originalAssetUrl) { + const urlObject = new URL(originalAssetUrl); + if (urlObject.hostname !== "pubpub.org") { + assetUrls[originalAssetUrl] = ""; + } + } + } + }); + await Promise.all( + Object.keys(assetUrls).map(async (originalAssetUrl) => { + try { + const assetData = await getAssetFile(originalAssetUrl); + if (assetData) { + const uploadedUrl = await uploadFileToS3( + pubId, + assetData.filename, + assetData.buffer, + { contentType: assetData.mimetype } + ); + assetUrls[originalAssetUrl] = uploadedUrl.replace( + "assets.app.pubpub.org.s3.us-east-1.amazonaws.com", + "assets.app.pubpub.org" + ); + } else { + assetUrls[originalAssetUrl] = originalAssetUrl; + } + } catch (err) { + assetUrls[originalAssetUrl] = originalAssetUrl; + } + }) + ); + + visit(tree, "element", (node: any) => { + const hasSrc = ["img", "video", "audio"].includes(node.tagName); + const isDownload = + node.tagName === "a" && node.properties.className === "file-button"; + if (hasSrc || isDownload) { + const propertyKey = hasSrc ? "src" : "href"; + const originalAssetUrl = node.properties[propertyKey]; + if (assetUrls[originalAssetUrl]) { + node.properties[propertyKey] = assetUrls[originalAssetUrl]; + } + } + }); + }) + .process(html); + return String(result); +}; const processHtml = async (html: string): Promise => { const result = await rehype() .use(structureFormatting) + .use(formatLists) .use(removeVerboseFormatting) .use(removeGoogleLinkForwards) .use(processLocalLinks) + .use(formatFigureReferences) /* Assumes figures are still tables */ .use(structureImages) .use(structureVideos) .use(structureAudio) @@ -53,8 +130,13 @@ const processHtml = async (html: string): Promise => { .use(structureCodeBlock) .use(structureInlineCode) .use(structureAnchors) + .use(structureTables) + .use(cleanUnusedSpans) .use(structureReferences) .use(structureFootnotes) + .use(appendFigureAttributes) /* Assumes figures are
elements */ + .use(removeEmptyFigCaption) + .use(removeDescription) .use(rehypeFormat) .process(html); return String(result); @@ -62,12 +144,33 @@ const processHtml = async (html: string): Promise => { export const formatDriveData = async ( dataFromDrive: DriveData, - communitySlug: string + communitySlug: string, + pubId: string, + createVersions: boolean ): Promise => { const formattedPubHtml = await processHtml(dataFromDrive.pubHtml); + const formattedPubHtmlWithAssets = await processAssets(formattedPubHtml, pubId); + if (!createVersions) { + return { + pubHtml: String(formattedPubHtmlWithAssets), + pubDescription: "", + versions: [], + discussions: [], + }; + } + /* Check for a description in the most recent version */ + const latestRawVersion = dataFromDrive.versions.reduce((latest, version) => { + return new Date(version.timestamp) > new Date(latest.timestamp) ? version : latest; + }, dataFromDrive.versions[0]); + + const latestPubDescription = latestRawVersion + ? getDescription(latestRawVersion.html) + : getDescription(dataFromDrive.pubHtml); + + /* Align versions to releases in legacy data and process HTML */ const releases: any = dataFromDrive.legacyData?.releases || []; - const findDescription = (timestamp: string) => { + const findVersionDescription = (timestamp: string) => { const matchingRelease = releases.find((release: any) => { return release.createdAt === timestamp; }); @@ -82,7 +185,7 @@ export const formatDriveData = async ( const versions = dataFromDrive.versions.map((version) => { const { timestamp, html } = version; const outputVersion: any = { - [`${communitySlug}:description`]: findDescription(timestamp), + [`${communitySlug}:description`]: findVersionDescription(timestamp), [`${communitySlug}:publication-date`]: timestamp, [`${communitySlug}:content`]: html, }; @@ -126,6 +229,38 @@ export const formatDriveData = async ( : comment.commenter && comment.commenter.orcid ? `https://orcid.org/${comment.commenter.orcid}` : null; + const convertDiscussionContent = (content: any) => { + const traverse = (node: any) => { + if (node.type === "image") { + return { ...node, attrs: { ...node.attrs, src: node.attrs.url } }; + } + if (node.type === "file") { + return { + type: "paragraph", + content: [ + { + text: node.attrs.fileName, + type: "text", + marks: [ + { type: "link", attrs: { href: node.attrs.url } }, + ], + }, + ], + }; + } + if (node.content) { + return { ...node, content: node.content.map(traverse) }; + } + return node; + }; + return traverse(content); + }; + const prosemirrorToMarkdown = (content: any): string => { + const convertedContent = convertDiscussionContent(content); + const doc = Node.fromJSON(schema, convertedContent); + return defaultMarkdownSerializer.serialize(doc); + }; + const markdownContent = prosemirrorToMarkdown(comment.content); const commentObject: any = { id: comment.id, values: { @@ -133,7 +268,8 @@ export const formatDriveData = async ( index === 0 && discussion.anchors.length ? JSON.stringify(discussion.anchors[0]) : undefined, - [`${communitySlug}:content`]: comment.text, + // [`${communitySlug}:content`]: comment.text, + [`${communitySlug}:content`]: markdownContent, [`${communitySlug}:publication-date`]: comment.createdAt, [`${communitySlug}:full-name`]: commentAuthorName, [`${communitySlug}:orcid`]: commentAuthorORCID, @@ -161,7 +297,8 @@ export const formatDriveData = async ( const comments = discussions ? flattenComments(discussions) : []; const output = { - pubHtml: String(formattedPubHtml), + pubDescription: latestPubDescription, + pubHtml: String(formattedPubHtmlWithAssets), versions, discussions: comments, }; diff --git a/core/actions/googleDriveImport/gdocPlugins.test.ts b/core/actions/googleDriveImport/gdocPlugins.test.ts index c737a2794..bb9466aba 100644 --- a/core/actions/googleDriveImport/gdocPlugins.test.ts +++ b/core/actions/googleDriveImport/gdocPlugins.test.ts @@ -4,8 +4,15 @@ import { expect, test } from "vitest"; import { logger } from "logger"; import { + appendFigureAttributes, basic, + cleanUnusedSpans, + formatFigureReferences, + formatLists, + getDescription, processLocalLinks, + removeDescription, + removeEmptyFigCaption, removeGoogleLinkForwards, removeVerboseFormatting, structureAnchors, @@ -21,6 +28,7 @@ import { structureInlineCode, structureInlineMath, structureReferences, + structureTables, structureVideos, tableToObjectArray, } from "./gdocPlugins"; @@ -73,6 +81,43 @@ test("Convert double table", async () => { expect(result).toStrictEqual(expectedOutput); }); +test("Convert vert table", async () => { + const inputNode = JSON.parse( + '{"type":"element","tagName":"table","properties":{},"children":[{"type":"element","tagName":"tbody","properties":{},"children":[{"type":"element","tagName":"tr","properties":{},"children":[{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"Type"}]}]}]},{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"Image"}]}]}]}]},{"type":"element","tagName":"tr","properties":{},"children":[{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"Id"}]}]}]},{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"n8r4ihxcrly"}]}]}]}]},{"type":"element","tagName":"tr","properties":{},"children":[{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"Source"}]}]}]},{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"https://resize-v3.pubpub.org/123"}]}]}]}]},{"type":"element","tagName":"tr","properties":{},"children":[{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"Alt Text"}]}]}]},{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"b","properties":{},"children":[{"type":"text","value":"123"}]}]}]}]},{"type":"element","tagName":"tr","properties":{},"children":[{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"Align"}]}]}]},{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"full"}]}]}]},{"type":"element","tagName":"tr","properties":{},"children":[{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"element","tagName":"span","properties":{},"children":[{"type":"text","value":"Size"}]}]}]},{"type":"element","tagName":"td","properties":{},"children":[{"type":"element","tagName":"p","properties":{},"children":[{"type":"text","value":"50"}]}]}]}]}]}' + ); + const expectedOutput = [ + { + type: "image", + id: "n8r4ihxcrly", + source: "https://resize-v3.pubpub.org/123", + alttext: "123", + align: "full", + size: "50", + }, + ]; + + const result = tableToObjectArray(inputNode); + + expect(result).toStrictEqual(expectedOutput); +}); + +test("Convert link-source table", async () => { + const inputNode = JSON.parse( + '{"type":"element","tagName":"table","children":[{"type":"element","tagName":"tbody","children":[{"type":"element","tagName":"tr","children":[{"type":"element","tagName":"td","children":[{"type":"element","tagName":"p","children":[{"type":"element","tagName":"span","children":[{"type":"text","value":"Type"}]}]}]},{"type":"element","tagName":"td","children":[{"type":"element","tagName":"p","children":[{"type":"text"},{"type":"element","tagName":"span","children":[{"type":"text","value":"Source"}]}]}]},{"type":"element","tagName":"td","children":[{"type":"element","tagName":"p","children":[{"type":"element","tagName":"span","children":[{"type":"text","value":"Static Image"}]}]}]}]},{"type":"element","tagName":"tr","children":[{"type":"element","tagName":"td","children":[{"type":"element","tagName":"p","children":[{"type":"element","tagName":"span","children":[{"type":"text","value":"Video"}]}]}]},{"type":"element","tagName":"td","children":[{"type":"element","tagName":"p","children":[{"type":"element","tagName":"span","children":[{"type":"element","tagName":"a","properties":{"href":"https://www.image-url.com"},"children":[{"type":"text","value":"image-filename.png"}]}]}]}]},{"type":"element","tagName":"td","children":[{"type":"element","tagName":"p","children":[{"type":"element","tagName":"span","children":[{"type":"element","tagName":"a","properties":{"href":"https://www.fallback-url.com"},"children":[{"type":"text","value":"fallback-filename.png"}]}]}]}]}]}]}]}' + ); + const expectedOutput = [ + { + source: "https://www.image-url.com", + type: "video", + staticimage: "https://www.fallback-url.com", + }, + ]; + + const result = tableToObjectArray(inputNode); + + expect(result).toStrictEqual(expectedOutput); +}); + test("Do Nothing", async () => { const inputHtml = '
Content
'; @@ -137,6 +182,8 @@ test("Structure Images", async () => {

Source

Caption

Alt Text

+

Align

+

Size

Image

@@ -144,6 +191,8 @@ test("Structure Images", async () => {

https://resize-v3.pubpub.org/123

With a caption. Bold

123

+

full

+

50

@@ -154,7 +203,7 @@ test("Structure Images", async () => { -
+
123

@@ -177,8 +226,153 @@ test("Structure Images", async () => { expect(trimAll(result)).toBe(trimAll(expectedOutputHtml)); }); +test("Structure Images - Vert Table", async () => { + const inputHtml = ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Type

Image

Id

n8r4ihxcrly

Source

https://resize-v3.pubpub.org/123

Caption

With a caption. Bold

Alt Text

123

Align

full

Size

50

+ + + `; + const expectedOutputHtml = ` + + + +

+ 123 +
+

+ With a caption. + Bold +

+
+
+ + + `; -test("Structure Images", async () => { + const result = await rehype() + .use(structureImages) + .process(inputHtml) + .then((file) => String(file)) + .catch((error) => { + logger.error(error); + }); + + expect(trimAll(result)).toBe(trimAll(expectedOutputHtml)); +}); +test("Structure Images - DoubleVert Table", async () => { + const inputHtml = ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Type

Image

Image

Id

n8r4ihxcrly

abr4ihxcrly

Source

https://resize-v3.pubpub.org/123

https://resize-v4.pubpub.org/123

Caption

With a caption. Bold

Alt Text

123

abc

Align

full

left

Size

50

75

+ + + `; + const expectedOutputHtml = ` + + + +
+ 123 +
+

+ With a caption. + Bold +

+
+
+
+ abc +

+
+ + + `; + + const result = await rehype() + .use(structureImages) + .process(inputHtml) + .then((file) => String(file)) + .catch((error) => { + logger.error(error); + }); + + expect(trimAll(result)).toBe(trimAll(expectedOutputHtml)); +}); + +test("Structure Videos", async () => { const inputHtml = ` @@ -191,6 +385,8 @@ test("Structure Images", async () => {

Source

Caption

Static Image

+

Align

+

Size

Video

@@ -198,6 +394,8 @@ test("Structure Images", async () => {

https://resize-v3.pubpub.org/123.mp4

With a caption. Bold

https://example.com +

full

+

50

@@ -208,7 +406,7 @@ test("Structure Images", async () => { -
+