diff --git a/.github/workflows/container-beemo-ghcr.yaml b/.github/workflows/container-beemo-ghcr.yaml new file mode 100644 index 000000000..ca2a0c9ff --- /dev/null +++ b/.github/workflows/container-beemo-ghcr.yaml @@ -0,0 +1,53 @@ +name: container-beemo-ghcr +on: + push: + branches: + - main +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + container-beemo-ghcr: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=beemo:,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/beemo/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-bigsky-aws.yaml b/.github/workflows/container-bigsky-aws.yaml new file mode 100644 index 000000000..5c1bbc163 --- /dev/null +++ b/.github/workflows/container-bigsky-aws.yaml @@ -0,0 +1,52 @@ +name: container-bigsky-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: bigsky + +jobs: + container-bigsky-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/bigsky/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-bigsky-ghcr.yaml b/.github/workflows/container-bigsky-ghcr.yaml new file mode 100644 index 000000000..2d37b9996 --- /dev/null +++ b/.github/workflows/container-bigsky-ghcr.yaml @@ -0,0 +1,53 @@ +name: container-bigsky-ghcr +on: [push] +env: + REGISTRY: ghcr.io + USERNAME: ${{ github.actor }} + PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + container-bigsky-ghcr: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=bigsky:,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/bigsky/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-bluepages-aws.yaml b/.github/workflows/container-bluepages-aws.yaml new file mode 100644 index 000000000..af37c5627 --- /dev/null +++ b/.github/workflows/container-bluepages-aws.yaml @@ -0,0 +1,52 @@ +name: container-bluepages-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: bluepages + +jobs: + container-bluepages-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/bluepages/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-collectiondir-aws.yaml b/.github/workflows/container-collectiondir-aws.yaml new file mode 100644 index 000000000..057553ad4 --- /dev/null +++ b/.github/workflows/container-collectiondir-aws.yaml @@ -0,0 +1,52 @@ +name: container-collectiondir-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: collectiondir + +jobs: + container-collectiondir-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/collectiondir/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-hepa-aws.yaml b/.github/workflows/container-hepa-aws.yaml new file mode 100644 index 000000000..336af4321 --- /dev/null +++ b/.github/workflows/container-hepa-aws.yaml @@ -0,0 +1,52 @@ +name: container-hepa-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: hepa + +jobs: + container-hepa-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/hepa/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-hepa-ghcr.yaml b/.github/workflows/container-hepa-ghcr.yaml new file mode 100644 index 000000000..bcb3269d9 --- /dev/null +++ b/.github/workflows/container-hepa-ghcr.yaml @@ -0,0 +1,54 @@ +name: container-hepa-ghcr +on: + push: + branches: + - main + - bnewbold/automod +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + container-hepa-ghcr: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=hepa:,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/hepa/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-palomar-aws.yaml b/.github/workflows/container-palomar-aws.yaml new file mode 100644 index 000000000..6c7cb4565 --- /dev/null +++ b/.github/workflows/container-palomar-aws.yaml @@ -0,0 +1,52 @@ +name: container-palomar-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: palomar + +jobs: + container-palomar-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/palomar/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-palomar-ghcr.yaml b/.github/workflows/container-palomar-ghcr.yaml new file mode 100644 index 000000000..7b6df603d --- /dev/null +++ b/.github/workflows/container-palomar-ghcr.yaml @@ -0,0 +1,53 @@ +name: container-palomar-ghcr +on: [push] +env: + REGISTRY: ghcr.io + USERNAME: ${{ github.actor }} + PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + container-palomar-ghcr: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=palomar:,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/palomar/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-rainbow-aws.yaml b/.github/workflows/container-rainbow-aws.yaml new file mode 100644 index 000000000..412be454a --- /dev/null +++ b/.github/workflows/container-rainbow-aws.yaml @@ -0,0 +1,52 @@ +name: container-rainbow-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: rainbow + +jobs: + container-rainbow-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/rainbow/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-relay-aws.yaml b/.github/workflows/container-relay-aws.yaml new file mode 100644 index 000000000..8afc12b65 --- /dev/null +++ b/.github/workflows/container-relay-aws.yaml @@ -0,0 +1,52 @@ +name: container-relay-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: relay + +jobs: + container-relay-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/relay/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-relay-ghcr.yaml b/.github/workflows/container-relay-ghcr.yaml new file mode 100644 index 000000000..6e5fcb4c4 --- /dev/null +++ b/.github/workflows/container-relay-ghcr.yaml @@ -0,0 +1,53 @@ +name: container-relay-ghcr +on: [push] +env: + REGISTRY: ghcr.io + USERNAME: ${{ github.actor }} + PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + container-bigsky-ghcr: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=relay:,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/relay/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-sonar-aws.yaml b/.github/workflows/container-sonar-aws.yaml new file mode 100644 index 000000000..df23509d4 --- /dev/null +++ b/.github/workflows/container-sonar-aws.yaml @@ -0,0 +1,52 @@ +name: container-sonar-aws +on: [push] +env: + REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }} + USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }} + PASSWORD: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_PASSWORD }} + # github.repository as / + IMAGE_NAME: sonar + +jobs: + container-sonar-aws: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/sonar/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-sonar-ghcr.yaml b/.github/workflows/container-sonar-ghcr.yaml new file mode 100644 index 000000000..22e0eefd2 --- /dev/null +++ b/.github/workflows/container-sonar-ghcr.yaml @@ -0,0 +1,53 @@ +name: container-sonar-ghcr +on: [push] +env: + REGISTRY: ghcr.io + USERNAME: ${{ github.actor }} + PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + container-bigsky-ghcr: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USERNAME }} + password: ${{ env.PASSWORD }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,enable=true,priority=100,prefix=sonar:,suffix=,format=long + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/sonar/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/container-tap-ghcr.yaml b/.github/workflows/container-tap-ghcr.yaml new file mode 100644 index 000000000..28cb880cf --- /dev/null +++ b/.github/workflows/container-tap-ghcr.yaml @@ -0,0 +1,58 @@ +name: container-tap-ghcr +on: + push: + branches: + - main + tags: + - 'tap-v*' +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + container-tap-ghcr: + if: github.repository == 'bluesky-social/indigo' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v1 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/tap + tags: | + type=sha,enable=true,priority=100,format=long + type=match,pattern=tap-v(\d+\.\d+\.\d+),group=1,enable=${{ startsWith(github.ref, 'refs/tags/tap-v') }} + type=match,pattern=tap-v(\d+\.\d+),group=1,enable=${{ startsWith(github.ref, 'refs/tags/tap-v') }} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/tap-v') }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v4 + with: + context: . + file: ./cmd/tap/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/golang.yml b/.github/workflows/golang.yml index 8276dafd6..159147912 100644 --- a/.github/workflows/golang.yml +++ b/.github/workflows/golang.yml @@ -1,15 +1,25 @@ name: golang -on: [push] + +on: + pull_request: + push: + branches: + - main + +concurrency: + group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}' + cancel-in-progress: true + jobs: build-and-test: runs-on: ubuntu-latest steps: - name: Git Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Set up Go tooling - uses: actions/setup-go@v3 + uses: actions/setup-go@v6 with: - go-version: 1.19 + go-version-file: go.mod - name: Build run: make build - name: Test @@ -18,10 +28,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Git Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Set up Go tooling - uses: actions/setup-go@v3 + uses: actions/setup-go@v6 with: - go-version: 1.19 + go-version-file: go.mod - name: Lint run: make lint diff --git a/.github/workflows/sync-internal.yaml b/.github/workflows/sync-internal.yaml new file mode 100644 index 000000000..b1bdd04a8 --- /dev/null +++ b/.github/workflows/sync-internal.yaml @@ -0,0 +1,31 @@ +name: Sync to internal repo + +on: + push: + branches: [main] + +jobs: + sync: + runs-on: ubuntu-latest + if: github.repository == 'bluesky-social/indigo' + steps: + - name: Checkout public repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Generate GitHub App Token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.SYNC_INTERNAL_APP_ID }} + private-key: ${{ secrets.SYNC_INTERNAL_PK }} + repositories: indigo-internal + - name: Push to internal repo + env: + TOKEN: ${{ steps.app-token.outputs.token }} + run: | + git config user.name "github-actions" + git config user.email "test@users.noreply.github.com" + git config --unset-all http.https://github.com/.extraheader + git remote add internal https://x-access-token:${TOKEN}@github.com/bluesky-social/indigo-internal.git + git push internal main --force diff --git a/.gitignore b/.gitignore index 76bf93634..e3ba180a4 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,42 @@ _build/ src/build/ *.log *.db +*.db-shm +*.db-wal +/data/ +test-coverage.out + +# executables +/beemo +/bigsky +/bluepages +/fakermaker +/gosky +/hepa +/lexgen +/palomar +/rainbow +/relay +/sonar +/sonar-cli +/stress +/supercollider +/tap # Don't ignore this file itself, or other specific dotfiles !.gitignore !.github/ +!.golangci.yaml + +# Don't commit your (default location) creds +bsky.auth + +# Sonar cursor file +sonar_cursor.json +out/ +state.json +netsync-out/ + +# Relay dash output +/public/ +/cmd/relay/public diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 000000000..aac246bdf --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,18 @@ +version: "2" +run: + modules-download-mode: readonly +issues: + max-issues-per-linter: 0 + max-same-issues: 0 + +linters: + disable: + - errcheck + - gocritic + - unused + settings: + errcheck: + check-type-assertions: true + disable-default-exclusions: true + gocritic: + enable-all: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 997f22153..335d133dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [d0ea072] - 2023-03-31 + +Large Lexicon refactor, and updates to streaming event wire schemas. The +relevant typescript repo pull request, with more details, is +. + + +## [61fc4c0] - 2023-03-10 + +MST and repo metadata schema refactor, resulting in repo binary format v2. + + ## [init] - 2023-01-19 Forked this repo from [whyrusleeping/gosky](https://github.com/whyrusleeping/gosky). diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 000000000..aea5fd895 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,176 @@ + +## git repo contents + +Run with, eg, `go run ./cmd/rainbow`): + +- `cmd/bigsky`: relay daemon +- `cmd/relay`: new (sync v1.1) relay daemon +- `cmd/palomar`: search indexer and query service (OpenSearch) +- `cmd/gosky`: client CLI for talking to a PDS +- `cmd/lexgen`: codegen tool for lexicons (Lexicon JSON to Go package) +- `cmd/stress`: connects to local/default PDS and creates a ton of random posts +- `cmd/beemo`: slack bot for moderation reporting (Bluesky Moderation Observer) +- `cmd/fakermaker`: helper to generate fake accounts and content for testing +- `cmd/supercollider`: event stream load generation tool +- `cmd/sonar`: event stream monitoring tool +- `cmd/hepa`: auto-moderation rule engine service +- `cmd/rainbow`: firehose fanout service +- `cmd/bluepages`: identity directory service +- `cmd/tap`: synchronization and backfill tool for atproto apps +- `gen`: dev tool to run CBOR type codegen + +Packages: + +- `api`: mostly output of lexgen (codegen) for lexicons: structs, CBOR marshaling. some higher-level code, and a PLC client (may rename) + - `api/atproto`: generated types for `com.atproto` lexicon + - `api/agnostic`: variants of `com.atproto` types which work better with unknown lexicon data + - `api/bsky`: generated types for `app.bsky` lexicon + - `api/chat`: generated types for `chat.bsky` lexicon + - `api/ozone`: generated types for `tools.ozone` lexicon +- `atproto/atcrypto`: cryptographic helpers (signing, key generation and serialization) +- `atproto/syntax`: string types and parsers for identifiers, datetimes, etc +- `atproto/identity`: DID and handle resolution +- `atproto/atdata`: helpers for atproto data as JSON or CBOR with unknown schema +- `atproto/lexicon`: lexicon validation of generic data +- `atproto/repo`: repo and MST implementation +- `automod`: moderation and anti-spam rules engine +- `bgs`: relay server implementation for crawling, etc (for bigsky implementation) +- `carstore`: library for storing repo data in CAR files on disk, plus a metadata SQL db +- `events`: types, codegen CBOR helpers, and persistence for event feeds +- `indexer`: aggregator, handling like counts etc in SQL database +- `lex`: implements codegen for Lexicons (!) +- `models`: database types/models/schemas; shared in several places +- `mst`: merkle search tree implementation +- `notifs`: helpers for notification objects (hydration, etc) +- `pds`: PDS server implementation +- `plc`: implementation of a *fake* PLC server (not persisted), and a PLC client +- `repo`: implements atproto repo on top of a blockstore. CBOR types +- `repomgr`: wraps many repos with a single carstore backend. handles events, locking +- `search`: search server implementation +- `testing`: integration tests; testing helpers +- `util`: a few common definitions (may rename) +- `xrpc`: XRPC client (not server) helpers + + +## Jargon + +- Relay: service which crawls/consumes content from "all" PDSs and re-broadcasts as a firehose +- BGS: Big Graph Service, previous name for what is now "Relay" +- PDS: Personal Data Server (or Service), which stores user atproto repositories and acts as a user agent in the network +- CLI: Command Line Tool +- CBOR: a binary serialization format, smilar to JSON +- PLC: "placeholder" DID provider, see +- DID: Decentralized IDentifier, a flexible W3C specification for persistent identifiers in URI form (eg, `did:plc:abcd1234`) +- XRPC: atproto convention for HTTP GET and POST endpoints specified by namespaced Lexicon schemas +- CAR: simple file format for storing binary content-addressed blocks/blobs, sort of like .tar files +- CID: content identifier for binary blobs, basically a flexible encoding of hash values +- MST: Merkle Search Tree, a key/value map data structure using content addressed nodes + + +## Lexicon and CBOR code generation + +`gen/main.go` has a list of types internal to packages in this repo which need CBOR helper codegen. If you edit those types, or update the listed types/packages, re-run codegen like: + + # make sure everything can build cleanly first + make build + + # then generate + go run ./gen + +To run codegen for new or updated Lexicons, using lexgen, first place (or git checkout) the JSON lexicon files at `../atproto/`. Then, in *this* repository (indigo), run commands like: + + go run ./cmd/lexgen/ --package bsky --prefix app.bsky --outdir api/bsky ../atproto/lexicons/app/bsky/ + go run ./cmd/lexgen/ --package atproto --prefix com.atproto --outdir api/atproto ../atproto/lexicons/com/atproto/ + +You may want to delete all the codegen files before re-generating, to detect deleted files. + +It can require some manual munging between the lexgen step and a later `go run ./gen` to make sure things compile at least temporarily; otherwise the `gen` will not run. In some cases, you might also need to add new types to `./gen/main.go`. + +To generate server stubs and handlers, push them in a temporary directory first, then merge changes in to the actual PDS code: + + mkdir tmppds + go run ./cmd/lexgen/ --package pds --gen-server --types-import com.atproto:github.com/bluesky-social/indigo/api/atproto --types-import app.bsky:github.com/bluesky-social/indigo/api/bsky --outdir tmppds --gen-handlers ../atproto/lexicons + + +## Tips and Tricks + +When debugging websocket streams, the `websocat` tool (rust) can be helpful. CBOR binary is sort of mangled in to text by default. Eg: + + # consume repo events from PDS + websocat ws://localhost:4989/events + + # consume repo events from Relay + websocat ws://localhost:2470/events + +Send the Relay a ding-dong: + + # tell Relay to consume from PDS + http --json post localhost:2470/add-target host="localhost:4989" + +Set the log level to be more verbose, using an env variable: + + GOLOG_LOG_LEVEL=info go run ./cmd/pds + + +## `gosky` basic usage + +Running against local typescript PDS in `dev-env` mode: + + # as "alice" user + go run ./cmd/gosky/ --pds-host http://localhost:2583 account create-session alice.test hunter2 > bsky.auth + +The `bsky.auth` file is the default place that `gosky` and other client commands will look for auth info. + + +## Integrated Development + +Sometimes it is helpful to run a PLC, PDS, Relay, and other components, all locally on your laptop, across languages. This section describes one setup for this. + +First, you need PostgreSQL running locally. This could be via docker, or the following commands assume some kind of debian/ubuntu setup with a postgres server package installed and running. + +Create a user and databases for PLC+PDS: + + # use 'yksb' as weak default password for local-only dev + sudo -u postgres createuser -P -s bsky + + sudo -u postgres createdb plc_dev -O bsky + sudo -u postgres createdb pds_dev -O bsky + +If you end up needing to wipe the databases: + + sudo -u postgres dropdb plc_dev + sudo -u postgres dropdb pds_dev + +Checkout the `did-method-plc` repo in on terminal and run: + + make run-dev-plc + +Checkout the `atproto` repo in another terminal and run: + + make run-dev-pds + +In this repo (indigo), start a Relay, in two separate terminals: + + make run-dev-relay + +In a final terminal, run fakermaker to inject data into the system: + + # setup and create initial accounts; 100 by default + mkdir data/fakermaker/ + export GOLOG_LOG_LEVEL=info + go run ./cmd/fakermaker/ gen-accounts > data/fakermaker/accounts.json + + # create or update profiles for all the accounts + go run ./cmd/fakermaker/ gen-profiles + + # create follow graph between accounts + go run ./cmd/fakermaker/ gen-graph + + # create posts, including mentions and image uploads + go run ./cmd/fakermaker/ gen-posts + + # create more interactions, such as likes, between accounts + go run ./cmd/fakermaker/ gen-interactions + + # lastly, read-only queries, including timelines, notifications, and post threads + go run ./cmd/fakermaker/ run-browsing diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE-DUAL b/LICENSE-DUAL new file mode 100644 index 000000000..d82d99b3f --- /dev/null +++ b/LICENSE-DUAL @@ -0,0 +1,8 @@ +Except as otherwise noted in individual files, this software is licensed under the MIT license (), or the Apache License, Version 2.0 (). + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. + +Copyright (c) 2022-2026 Bluesky Social PBC, @whyrusleeping, and Contributors + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/LICENSE.md b/LICENSE-MIT similarity index 88% rename from LICENSE.md rename to LICENSE-MIT index c18b4b7d1..9cf106272 100644 --- a/LICENSE.md +++ b/LICENSE-MIT @@ -1,4 +1,4 @@ -Copyright 2022 @whyrusleeping +MIT License Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -7,13 +7,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index 6f0e30857..eb122f626 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,13 @@ +SHELL = /bin/bash +.SHELLFLAGS = -o pipefail -c + +# base path for Lexicon document tree (for lexgen) +LEXDIR?=./lexicons + +# https://github.com/golang/go/wiki/LoopvarExperiment +export GOEXPERIMENT := loopvar + .PHONY: help help: ## Print info about all commands @echo "Commands:" @@ -7,18 +16,141 @@ help: ## Print info about all commands .PHONY: build build: ## Build all executables - go build ./... + go build ./cmd/gosky + go build ./cmd/bigsky + go build ./cmd/relay + go build ./cmd/beemo + go build ./cmd/lexgen + go build ./cmd/stress + go build ./cmd/fakermaker + go build ./cmd/hepa + go build ./cmd/supercollider + go build -o ./sonar-cli ./cmd/sonar + go build ./cmd/palomar + go build ./cmd/tap + +.PHONY: all +all: build .PHONY: test -test: build ## Run all tests +test: ## Run tests go test ./... +.PHONY: test-short +test-short: ## Run tests, skipping slower integration tests + go test -test.short ./... + +.PHONY: test-interop +test-interop: ## Run tests, including local interop (requires services running) + go clean -testcache && go test -tags=localinterop ./... + +.PHONY: test-search +test-search: ## Run tests, including local search indexing (requires services running) + go clean -testcache && go test -tags=localsearch ./... + +.PHONY: coverage-html +coverage-html: ## Generate test coverage report and open in browser + go test ./... -coverpkg=./... -coverprofile=test-coverage.out + go tool cover -html=test-coverage.out + .PHONY: lint -lint: ## Run style checks and verify syntax - go vet -a ./... +lint: ## Verify code style and run static checks + go vet ./... test -z $(gofmt -l ./...) - # TODO: golangci-lint .PHONY: fmt -fmt: ## Run syntax re-formatting +fmt: ## Run syntax re-formatting (modify in place) go fmt ./... + +.PHONY: check +check: ## Compile everything, checking syntax (does not output binaries) + go build ./... + +.PHONY: lexgen +lexgen: ## Run codegen tool for lexicons (lexicon JSON to Go packages) + go run ./cmd/lexgen/ --build-file cmd/lexgen/bsky.json $(LEXDIR) + +.PHONY: cborgen +cborgen: ## Run codegen tool for CBOR serialization + go run ./gen + +.env: + if [ ! -f ".env" ]; then cp example.dev.env .env; fi + +.PHONY: run-postgres +run-postgres: .env ## Runs a local postgres instance + docker compose -f cmd/bigsky/docker-compose.yml up -d + +.PHONY: run-dev-opensearch +run-dev-opensearch: .env ## Runs a local opensearch instance + docker build -f cmd/palomar/Dockerfile.opensearch . -t opensearch-palomar + docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e "OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penSearch-Pal0mar" opensearch-palomar + +.PHONY: run-dev-relay +run-dev-relay: .env ## Runs relay for local dev + LOG_LEVEL=info go run ./cmd/relay --admin-password localdev serve + +.PHONY: run-dev-ident +run-dev-ident: .env ## Runs 'bluepages' identity directory for local dev + GOLOG_LOG_LEVEL=info go run ./cmd/bluepages serve + +.PHONY: build-relay-image +build-relay-image: ## Builds relay docker image + docker build -t relay -f cmd/relay/Dockerfile . + +.PHONY: build-relay-admin-ui +build-relay-admin-ui: ## Build relay admin web UI + cd cmd/relay/relay-admin-ui; yarn install --frozen-lockfile; yarn build + mkdir -p public + cp -r cmd/relay/relay-admin-ui/dist/* public/ + +.PHONY: run-relay-image +run-relay-image: + docker run -p 2470:2470 relay /relay serve --admin-password localdev +# --crawl-insecure-ws + +.PHONY: run-dev-search +run-dev-search: .env ## Runs search daemon for local dev + GOLOG_LOG_LEVEL=info go run ./cmd/palomar run + +.PHONY: sonar-up +sonar-up: # Runs sonar docker container + docker compose -f cmd/sonar/docker-compose.yml up --build -d || docker-compose -f cmd/sonar/docker-compose.yml up --build -d + +.PHONY: sc-reload +sc-reload: # Reloads supercollider + go run cmd/supercollider/main.go \ + reload \ + --port 6125 --total-events 2000000 \ + --hostname alpha.supercollider.jazco.io \ + --key-file out/alpha.pem \ + --output-file out/alpha_in.cbor + +.PHONY: sc-fire +sc-fire: # Fires supercollider + go run cmd/supercollider/main.go \ + fire \ + --port 6125 --events-per-second 10000 \ + --hostname alpha.supercollider.jazco.io \ + --key-file out/alpha.pem \ + --input-file out/alpha_in.cbor + +.PHONY: run-netsync +run-netsync: .env ## Runs netsync for local dev + go run ./cmd/netsync --checkout-limit 100 --worker-count 100 --out-dir ../netsync-out + +SCYLLA_VERSION := latest +SCYLLA_CPU := 0 +SCYLLA_NODES := 127.0.0.1:9042 + +.PHONY: run-scylla +run-scylla: + @echo "==> Running test instance of Scylla $(SCYLLA_VERSION)" + @docker pull scylladb/scylla:$(SCYLLA_VERSION) + @docker run --name scylla -p 9042:9042 --cpuset-cpus=$(SCYLLA_CPU) --memory 1G --rm -d scylladb/scylla:$(SCYLLA_VERSION) + @until docker exec scylla cqlsh -e "DESCRIBE SCHEMA"; do sleep 2; done + +.PHONY: stop-scylla +stop-scylla: + @echo "==> Stopping test instance of Scylla $(SCYLLA_VERSION)" + @docker stop scylla diff --git a/README.md b/README.md index 974d801fd..829141ed8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,123 @@ -# gosky +![photo](https://static.bnewbold.net/tmp/indigo_serac.jpeg) -A collection of utilities for interacting with atproto +# indigo: atproto libraries and services in golang +Some Bluesky software is developed in Typescript, and lives in the [bluesky-social/atproto](https://github.com/bluesky-social/atproto) repository. Some is developed in Go, and lives here. +**If you are not a Go developer and you want to run one of these tools**, you can do: +```bash +# with [Homebrew](brew.sh) installed +brew install go +# for example, to run tap +go install github.com/bluesky-social/indigo/cmd/tap +tap +``` + +Go will fetch dependencies, compile, and install `tap` or another service with a one-line `go install` command. + +*Soon*, we plan to decouple the tools in this repo so you can install them individually like [goat](https://formulae.brew.sh/formula/goat). + +## What is in here? + +**Go Services:** + +- **tap** ([README](./cmd/tap/README.md)): synchronization and backfill tool for atproto apps +- **relay** ([README](./cmd/relay/README.md)): relay reference implementation +- **rainbow** ([README](./cmd/rainbow/README.md)): firehose "splitter" or "fan-out" service +- **hepa** ([README](./cmd/hepa/README.md)): auto-moderation bot for [Ozone](https://ozone.tools) +- **palomar** ([README](./cmd/palomar/README.md)): fulltext search service for + +**Developer Tools:** + +- **goat** ([README](https://github.com/bluesky-social/goat)): CLI for interacting with network: CAR files, firehose, APIs, etc (moved to [separate repo](https://github.com/bluesky-social/goat)) + +**Go Packages:** + +> ⚠️ All the packages in this repository are under active development. Features and software interfaces have not stabilized and may break or be removed. + +| Package | Docs | +| ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `api/atproto`: generated types for `com.atproto.*` Lexicons | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/api/atproto)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/api/atproto) | +| `api/bsky`: generated types for `app.bsky.*` Lexicons | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/api/bsky)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/api/bsky) | +| `atproto/atclient`: HTTP API client | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/atclient)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/atclient) | +| `atproto/auth/oauth`: AT OAuth client | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/auth/oauth)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/auth/oauth)| +| `atproto/identity`: DID and handle resolution | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/identity)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/identity) | +| `atproto/syntax`: string types and parsers for identifiers | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/syntax)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/syntax) | +| `atproto/lexicon`: schema validation of data | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/lexicon)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/lexicon) | +| `atproto/repo`: repository data structure | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/repo)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/repo) | +| `atproto/repo/mst`: Merkle Search Tree implementation | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/repo/mst)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/repo/mst) | +| `atproto/atcrypto`: cryptographic signing and key serialization | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/atcrypto)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/atcrypto)| + +The TypeScript reference implementation, including PDS and bsky AppView services, is at [bluesky-social/atproto](https://github.com/bluesky-social/atproto). Source code for the Bluesky Social client app (for web and mobile) can be found at [bluesky-social/social-app](https://github.com/bluesky-social/social-app). + +## Development Quickstart + +First, you will need the Go toolchain installed. We develop using the latest stable version of the language. + +The Makefile provides wrapper commands for basic development: + + make build + make test + make fmt + make lint + +Individual commands can be run like: + + go run ./cmd/relay + +The [HACKING](./HACKING.md) file has a list of commands and packages in this repository and some other development tips. + +## What is atproto? + +_not to be confused with the [AT command set](https://en.wikipedia.org/wiki/Hayes_command_set) or [Adenosine triphosphate](https://en.wikipedia.org/wiki/Adenosine_triphosphate)_ + +The Authenticated Transfer Protocol ("ATP" or "atproto") is a decentralized social media protocol, developed by [Bluesky Social PBC](https://bsky.social). Learn more at: + +- [Overview and Guides](https://atproto.com/guides/overview) 👈 Best starting point +- [Github Discussions](https://github.com/bluesky-social/atproto/discussions) 👈 Great place to ask questions +- [Protocol Specifications](https://atproto.com/specs/atp) +- [Blogpost on self-authenticating data structures](https://bsky.social/about/blog/3-6-2022-a-self-authenticating-social-protocol) + +The Bluesky Social application encompasses a set of schemas and APIs built in the overall AT Protocol framework. The namespace for these "Lexicons" is `app.bsky.*`. + +## Contributions + +> While we do accept contributions, we prioritize high quality issues and pull requests. Adhering to the below guidelines will ensure a more timely review. + +**Rules:** + +- We may not respond to your issue or PR. +- We may close an issue or PR without much feedback. +- We may lock discussions or contributions if our attention is getting DDOSed. +- We do not provide support for build issues. + +**Guidelines:** + +- Check for existing issues before filing a new one, please. +- Open an issue and give some time for discussion before submitting a PR. +- Issues are for bugs & feature requests related to the golang implementation of atproto and related services. + - For high-level discussions, please use the [Discussion Forum](https://github.com/bluesky-social/atproto/discussions). + - For client issues, please use the relevant [social-app](https://github.com/bluesky-social/social-app) repo. +- Stay away from PRs that: + - Refactor large parts of the codebase + - Add entirely new features without prior discussion + - Change the tooling or libraries used without prior discussion + - Introduce new unnecessary dependencies + +Remember, we serve a wide community of users. Our day-to-day involves us constantly asking "which top priority is our top priority." If you submit well-written PRs that solve problems concisely, that's an awesome contribution. Otherwise, as much as we'd love to accept your ideas and contributions, we really don't have the bandwidth. + +## Are you a developer interested in building on atproto? + +Bluesky is an open social network built on the AT Protocol, a flexible technology that will never lock developers out of the ecosystems that they help build. With atproto, third-party can be as seamless as first-party through custom feeds, federated services, clients, and more. + +## License + +This project is dual-licensed under MIT and Apache 2.0 terms: + +- MIT license ([LICENSE-MIT](https://github.com/bluesky-social/indigo/blob/main/LICENSE-MIT) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0, ([LICENSE-APACHE](https://github.com/bluesky-social/indigo/blob/main/LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) + +Downstream projects and end users may chose either license individually, or both together, at their discretion. The motivation for this dual-licensing is the additional software patent assurance provided by Apache 2.0. + +Bluesky Social PBC has committed to a software patent non-aggression pledge. For details see [the original announcement](https://bsky.social/about/blog/10-01-2025-patent-pledge). diff --git a/api/agnostic/actorgetPreferences.go b/api/agnostic/actorgetPreferences.go new file mode 100644 index 000000000..150e3e798 --- /dev/null +++ b/api/agnostic/actorgetPreferences.go @@ -0,0 +1,28 @@ +// Copied from indigo:api/bsky/actorgetPreferences.go + +package agnostic + +// schema: app.bsky.actor.getPreferences + +import ( + "context" + + "github.com/bluesky-social/indigo/lex/util" +) + +// ActorGetPreferences_Output is the output of a app.bsky.actor.getPreferences call. +type ActorGetPreferences_Output struct { + Preferences []map[string]any `json:"preferences" cborgen:"preferences"` +} + +// ActorGetPreferences calls the XRPC method "app.bsky.actor.getPreferences". +func ActorGetPreferences(ctx context.Context, c util.LexClient) (*ActorGetPreferences_Output, error) { + var out ActorGetPreferences_Output + + params := map[string]interface{}{} + if err := c.LexDo(ctx, util.Query, "", "app.bsky.actor.getPreferences", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/agnostic/actorputPreferences.go b/api/agnostic/actorputPreferences.go new file mode 100644 index 000000000..02276da29 --- /dev/null +++ b/api/agnostic/actorputPreferences.go @@ -0,0 +1,25 @@ +// Copied from indigo:api/bsky/actorputPreferences.go + +package agnostic + +// schema: app.bsky.actor.putPreferences + +import ( + "context" + + "github.com/bluesky-social/indigo/lex/util" +) + +// ActorPutPreferences_Input is the input argument to a app.bsky.actor.putPreferences call. +type ActorPutPreferences_Input struct { + Preferences []map[string]any `json:"preferences" cborgen:"preferences"` +} + +// ActorPutPreferences calls the XRPC method "app.bsky.actor.putPreferences". +func ActorPutPreferences(ctx context.Context, c util.LexClient, input *ActorPutPreferences_Input) error { + if err := c.LexDo(ctx, util.Procedure, "application/json", "app.bsky.actor.putPreferences", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/agnostic/doc.go b/api/agnostic/doc.go new file mode 100644 index 000000000..66093f832 --- /dev/null +++ b/api/agnostic/doc.go @@ -0,0 +1,4 @@ +// Package indigo/api/agnostic provides schema-agnostic helpers for fetching records from the network. +// +// These are variants of endpoints in indigo/api/atproto. +package agnostic diff --git a/api/agnostic/identitygetRecommendedDidCredentials.go b/api/agnostic/identitygetRecommendedDidCredentials.go new file mode 100644 index 000000000..85fabcf6b --- /dev/null +++ b/api/agnostic/identitygetRecommendedDidCredentials.go @@ -0,0 +1,23 @@ +// Copied from indigo:api/atproto/identitygetRecommendedDidCredentials.go + +package agnostic + +// schema: com.atproto.identity.getRecommendedDidCredentials + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/lex/util" +) + +// IdentityGetRecommendedDidCredentials calls the XRPC method "com.atproto.identity.getRecommendedDidCredentials". +func IdentityGetRecommendedDidCredentials(ctx context.Context, c util.LexClient) (*json.RawMessage, error) { + var out json.RawMessage + + if err := c.LexDo(ctx, util.Query, "", "com.atproto.identity.getRecommendedDidCredentials", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/agnostic/identitysignPlcOperation.go b/api/agnostic/identitysignPlcOperation.go new file mode 100644 index 000000000..44c6f7328 --- /dev/null +++ b/api/agnostic/identitysignPlcOperation.go @@ -0,0 +1,38 @@ +// Copied from indigo:api/atproto/identitysignPlcOperation.go + +package agnostic + +// schema: com.atproto.identity.signPlcOperation + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/lex/util" +) + +// IdentitySignPlcOperation_Input is the input argument to a com.atproto.identity.signPlcOperation call. +type IdentitySignPlcOperation_Input struct { + AlsoKnownAs []string `json:"alsoKnownAs,omitempty" cborgen:"alsoKnownAs,omitempty"` + RotationKeys []string `json:"rotationKeys,omitempty" cborgen:"rotationKeys,omitempty"` + Services *json.RawMessage `json:"services,omitempty" cborgen:"services,omitempty"` + // token: A token received through com.atproto.identity.requestPlcOperationSignature + Token *string `json:"token,omitempty" cborgen:"token,omitempty"` + VerificationMethods *json.RawMessage `json:"verificationMethods,omitempty" cborgen:"verificationMethods,omitempty"` +} + +// IdentitySignPlcOperation_Output is the output of a com.atproto.identity.signPlcOperation call. +type IdentitySignPlcOperation_Output struct { + // operation: A signed DID PLC operation. + Operation *json.RawMessage `json:"operation" cborgen:"operation"` +} + +// IdentitySignPlcOperation calls the XRPC method "com.atproto.identity.signPlcOperation". +func IdentitySignPlcOperation(ctx context.Context, c util.LexClient, input *IdentitySignPlcOperation_Input) (*IdentitySignPlcOperation_Output, error) { + var out IdentitySignPlcOperation_Output + if err := c.LexDo(ctx, util.Procedure, "application/json", "com.atproto.identity.signPlcOperation", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/agnostic/identitysubmitPlcOperation.go b/api/agnostic/identitysubmitPlcOperation.go new file mode 100644 index 000000000..ef7e8a0af --- /dev/null +++ b/api/agnostic/identitysubmitPlcOperation.go @@ -0,0 +1,26 @@ +// Copied from indigo:api/atproto/identitysubmitPlcOperation.go + +package agnostic + +// schema: com.atproto.identity.submitPlcOperation + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/lex/util" +) + +// IdentitySubmitPlcOperation_Input is the input argument to a com.atproto.identity.submitPlcOperation call. +type IdentitySubmitPlcOperation_Input struct { + Operation *json.RawMessage `json:"operation" cborgen:"operation"` +} + +// IdentitySubmitPlcOperation calls the XRPC method "com.atproto.identity.submitPlcOperation". +func IdentitySubmitPlcOperation(ctx context.Context, c util.LexClient, input *IdentitySubmitPlcOperation_Input) error { + if err := c.LexDo(ctx, util.Procedure, "application/json", "com.atproto.identity.submitPlcOperation", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/agnostic/repoapplyWrites.go b/api/agnostic/repoapplyWrites.go new file mode 100644 index 000000000..0dbe127b8 --- /dev/null +++ b/api/agnostic/repoapplyWrites.go @@ -0,0 +1,188 @@ +// Copied from indigo:api/atproto/repoapplyWrites.go + +package agnostic + +// schema: com.atproto.repo.applyWrites + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/bluesky-social/indigo/lex/util" +) + +// RepoApplyWrites_Create is a "create" in the com.atproto.repo.applyWrites schema. +// +// Operation which creates a new record. +// +// RECORDTYPE: RepoApplyWrites_Create +type RepoApplyWrites_Create struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#create" cborgen:"$type,const=com.atproto.repo.applyWrites#create"` + Collection string `json:"collection" cborgen:"collection"` + Rkey *string `json:"rkey,omitempty" cborgen:"rkey,omitempty"` + Value *json.RawMessage `json:"value" cborgen:"value"` +} + +// RepoApplyWrites_CreateResult is a "createResult" in the com.atproto.repo.applyWrites schema. +// +// RECORDTYPE: RepoApplyWrites_CreateResult +type RepoApplyWrites_CreateResult struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#createResult" cborgen:"$type,const=com.atproto.repo.applyWrites#createResult"` + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + +// RepoApplyWrites_Delete is a "delete" in the com.atproto.repo.applyWrites schema. +// +// Operation which deletes an existing record. +// +// RECORDTYPE: RepoApplyWrites_Delete +type RepoApplyWrites_Delete struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#delete" cborgen:"$type,const=com.atproto.repo.applyWrites#delete"` + Collection string `json:"collection" cborgen:"collection"` + Rkey string `json:"rkey" cborgen:"rkey"` +} + +// RepoApplyWrites_DeleteResult is a "deleteResult" in the com.atproto.repo.applyWrites schema. +// +// RECORDTYPE: RepoApplyWrites_DeleteResult +type RepoApplyWrites_DeleteResult struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#deleteResult" cborgen:"$type,const=com.atproto.repo.applyWrites#deleteResult"` +} + +// RepoApplyWrites_Input is the input argument to a com.atproto.repo.applyWrites call. +type RepoApplyWrites_Input struct { + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // swapCommit: If provided, the entire operation will fail if the current repo commit CID does not match this value. Used to prevent conflicting repo mutations. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // validate: Can be set to 'false' to skip Lexicon schema validation of record data across all operations, 'true' to require it, or leave unset to validate only for known Lexicons. + Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` + Writes []*RepoApplyWrites_Input_Writes_Elem `json:"writes" cborgen:"writes"` +} + +type RepoApplyWrites_Input_Writes_Elem struct { + RepoApplyWrites_Create *RepoApplyWrites_Create + RepoApplyWrites_Update *RepoApplyWrites_Update + RepoApplyWrites_Delete *RepoApplyWrites_Delete +} + +func (t *RepoApplyWrites_Input_Writes_Elem) MarshalJSON() ([]byte, error) { + if t.RepoApplyWrites_Create != nil { + t.RepoApplyWrites_Create.LexiconTypeID = "com.atproto.repo.applyWrites#create" + return json.Marshal(t.RepoApplyWrites_Create) + } + if t.RepoApplyWrites_Update != nil { + t.RepoApplyWrites_Update.LexiconTypeID = "com.atproto.repo.applyWrites#update" + return json.Marshal(t.RepoApplyWrites_Update) + } + if t.RepoApplyWrites_Delete != nil { + t.RepoApplyWrites_Delete.LexiconTypeID = "com.atproto.repo.applyWrites#delete" + return json.Marshal(t.RepoApplyWrites_Delete) + } + return nil, fmt.Errorf("cannot marshal empty enum") +} +func (t *RepoApplyWrites_Input_Writes_Elem) UnmarshalJSON(b []byte) error { + typ, err := util.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.repo.applyWrites#create": + t.RepoApplyWrites_Create = new(RepoApplyWrites_Create) + return json.Unmarshal(b, t.RepoApplyWrites_Create) + case "com.atproto.repo.applyWrites#update": + t.RepoApplyWrites_Update = new(RepoApplyWrites_Update) + return json.Unmarshal(b, t.RepoApplyWrites_Update) + case "com.atproto.repo.applyWrites#delete": + t.RepoApplyWrites_Delete = new(RepoApplyWrites_Delete) + return json.Unmarshal(b, t.RepoApplyWrites_Delete) + + default: + return fmt.Errorf("closed enums must have a matching value") + } +} + +// RepoApplyWrites_Output is the output of a com.atproto.repo.applyWrites call. +type RepoApplyWrites_Output struct { + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Results []*RepoApplyWrites_Output_Results_Elem `json:"results,omitempty" cborgen:"results,omitempty"` +} + +type RepoApplyWrites_Output_Results_Elem struct { + RepoApplyWrites_CreateResult *RepoApplyWrites_CreateResult + RepoApplyWrites_UpdateResult *RepoApplyWrites_UpdateResult + RepoApplyWrites_DeleteResult *RepoApplyWrites_DeleteResult +} + +func (t *RepoApplyWrites_Output_Results_Elem) MarshalJSON() ([]byte, error) { + if t.RepoApplyWrites_CreateResult != nil { + t.RepoApplyWrites_CreateResult.LexiconTypeID = "com.atproto.repo.applyWrites#createResult" + return json.Marshal(t.RepoApplyWrites_CreateResult) + } + if t.RepoApplyWrites_UpdateResult != nil { + t.RepoApplyWrites_UpdateResult.LexiconTypeID = "com.atproto.repo.applyWrites#updateResult" + return json.Marshal(t.RepoApplyWrites_UpdateResult) + } + if t.RepoApplyWrites_DeleteResult != nil { + t.RepoApplyWrites_DeleteResult.LexiconTypeID = "com.atproto.repo.applyWrites#deleteResult" + return json.Marshal(t.RepoApplyWrites_DeleteResult) + } + return nil, fmt.Errorf("cannot marshal empty enum") +} +func (t *RepoApplyWrites_Output_Results_Elem) UnmarshalJSON(b []byte) error { + typ, err := util.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.repo.applyWrites#createResult": + t.RepoApplyWrites_CreateResult = new(RepoApplyWrites_CreateResult) + return json.Unmarshal(b, t.RepoApplyWrites_CreateResult) + case "com.atproto.repo.applyWrites#updateResult": + t.RepoApplyWrites_UpdateResult = new(RepoApplyWrites_UpdateResult) + return json.Unmarshal(b, t.RepoApplyWrites_UpdateResult) + case "com.atproto.repo.applyWrites#deleteResult": + t.RepoApplyWrites_DeleteResult = new(RepoApplyWrites_DeleteResult) + return json.Unmarshal(b, t.RepoApplyWrites_DeleteResult) + + default: + return fmt.Errorf("closed enums must have a matching value") + } +} + +// RepoApplyWrites_Update is a "update" in the com.atproto.repo.applyWrites schema. +// +// Operation which updates an existing record. +// +// RECORDTYPE: RepoApplyWrites_Update +type RepoApplyWrites_Update struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#update" cborgen:"$type,const=com.atproto.repo.applyWrites#update"` + Collection string `json:"collection" cborgen:"collection"` + Rkey string `json:"rkey" cborgen:"rkey"` + Value *json.RawMessage `json:"value" cborgen:"value"` +} + +// RepoApplyWrites_UpdateResult is a "updateResult" in the com.atproto.repo.applyWrites schema. +// +// RECORDTYPE: RepoApplyWrites_UpdateResult +type RepoApplyWrites_UpdateResult struct { + LexiconTypeID string `json:"$type,const=com.atproto.repo.applyWrites#updateResult" cborgen:"$type,const=com.atproto.repo.applyWrites#updateResult"` + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + +// RepoApplyWrites calls the XRPC method "com.atproto.repo.applyWrites". +func RepoApplyWrites(ctx context.Context, c util.LexClient, input *RepoApplyWrites_Input) (*RepoApplyWrites_Output, error) { + var out RepoApplyWrites_Output + if err := c.LexDo(ctx, util.Procedure, "application/json", "com.atproto.repo.applyWrites", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/agnostic/repocreateRecord.go b/api/agnostic/repocreateRecord.go new file mode 100644 index 000000000..44680b76e --- /dev/null +++ b/api/agnostic/repocreateRecord.go @@ -0,0 +1,51 @@ +// Copied from indigo:api/atproto/repocreateRecords.go + +package agnostic + +// schema: com.atproto.repo.createRecord + +import ( + "context" + + "github.com/bluesky-social/indigo/lex/util" +) + +// RepoDefs_CommitMeta is a "commitMeta" in the com.atproto.repo.defs schema. +type RepoDefs_CommitMeta struct { + Cid string `json:"cid" cborgen:"cid"` + Rev string `json:"rev" cborgen:"rev"` +} + +// RepoCreateRecord_Input is the input argument to a com.atproto.repo.createRecord call. +type RepoCreateRecord_Input struct { + // collection: The NSID of the record collection. + Collection string `json:"collection" cborgen:"collection"` + // record: The record itself. Must contain a $type field. + Record map[string]any `json:"record" cborgen:"record"` + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // rkey: The Record Key. + Rkey *string `json:"rkey,omitempty" cborgen:"rkey,omitempty"` + // swapCommit: Compare and swap with the previous commit by CID. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // validate: Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons. + Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` +} + +// RepoCreateRecord_Output is the output of a com.atproto.repo.createRecord call. +type RepoCreateRecord_Output struct { + Cid string `json:"cid" cborgen:"cid"` + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + +// RepoCreateRecord calls the XRPC method "com.atproto.repo.createRecord". +func RepoCreateRecord(ctx context.Context, c util.LexClient, input *RepoCreateRecord_Input) (*RepoCreateRecord_Output, error) { + var out RepoCreateRecord_Output + if err := c.LexDo(ctx, util.Procedure, "application/json", "com.atproto.repo.createRecord", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/agnostic/repogetRecord.go b/api/agnostic/repogetRecord.go new file mode 100644 index 000000000..b1aebe22b --- /dev/null +++ b/api/agnostic/repogetRecord.go @@ -0,0 +1,44 @@ +// Copied from indigo:api/atproto/repolistRecords.go + +package agnostic + +// schema: com.atproto.repo.getRecord + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/lex/util" +) + +// RepoGetRecord_Output is the output of a com.atproto.repo.getRecord call. +type RepoGetRecord_Output struct { + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + // NOTE: changed from lex decoder to json.RawMessage + Value *json.RawMessage `json:"value" cborgen:"value"` +} + +// RepoGetRecord calls the XRPC method "com.atproto.repo.getRecord". +// +// cid: The CID of the version of the record. If not specified, then return the most recent version. +// collection: The NSID of the record collection. +// repo: The handle or DID of the repo. +// rkey: The Record Key. +func RepoGetRecord(ctx context.Context, c util.LexClient, cid string, collection string, repo string, rkey string) (*RepoGetRecord_Output, error) { + var out RepoGetRecord_Output + + params := map[string]interface{}{ + "collection": collection, + "repo": repo, + "rkey": rkey, + } + if cid != "" { + params["cid"] = cid + } + if err := c.LexDo(ctx, util.Query, "", "com.atproto.repo.getRecord", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/agnostic/repolistRecords.go b/api/agnostic/repolistRecords.go new file mode 100644 index 000000000..7b854c882 --- /dev/null +++ b/api/agnostic/repolistRecords.go @@ -0,0 +1,56 @@ +// Copied from indigo:api/atproto/repolistRecords.go + +package agnostic + +// schema: com.atproto.repo.listRecords + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/lex/util" +) + +// RepoListRecords_Output is the output of a com.atproto.repo.listRecords call. +type RepoListRecords_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Records []*RepoListRecords_Record `json:"records" cborgen:"records"` +} + +// RepoListRecords_Record is a "record" in the com.atproto.repo.listRecords schema. +type RepoListRecords_Record struct { + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` + // NOTE: changed from lex decoder to json.RawMessage + Value *json.RawMessage `json:"value" cborgen:"value"` +} + +// RepoListRecords calls the XRPC method "com.atproto.repo.listRecords". +// +// collection: The NSID of the record type. +// limit: The number of records to return. +// repo: The handle or DID of the repo. +// reverse: Flag to reverse the order of the returned records. +func RepoListRecords(ctx context.Context, c util.LexClient, collection string, cursor string, limit int64, repo string, reverse bool) (*RepoListRecords_Output, error) { + var out RepoListRecords_Output + + params := map[string]interface{}{ + "collection": collection, + "repo": repo, + } + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if reverse != false { + params["reverse"] = reverse + } + + if err := c.LexDo(ctx, util.Query, "", "com.atproto.repo.listRecords", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/agnostic/repoputRecord.go b/api/agnostic/repoputRecord.go new file mode 100644 index 000000000..0d2d80c40 --- /dev/null +++ b/api/agnostic/repoputRecord.go @@ -0,0 +1,47 @@ +// Copied from indigo:api/atproto/repoputRecords.go + +package agnostic + +// schema: com.atproto.repo.putRecord + +import ( + "context" + + "github.com/bluesky-social/indigo/lex/util" +) + +// RepoPutRecord_Input is the input argument to a com.atproto.repo.putRecord call. +type RepoPutRecord_Input struct { + // collection: The NSID of the record collection. + Collection string `json:"collection" cborgen:"collection"` + // record: The record to write. + Record map[string]any `json:"record" cborgen:"record"` + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // rkey: The Record Key. + Rkey string `json:"rkey" cborgen:"rkey"` + // swapCommit: Compare and swap with the previous commit by CID. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // swapRecord: Compare and swap with the previous record by CID. WARNING: nullable and optional field; may cause problems with golang implementation + SwapRecord *string `json:"swapRecord,omitempty" cborgen:"swapRecord,omitempty"` + // validate: Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons. + Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` +} + +// RepoPutRecord_Output is the output of a com.atproto.repo.putRecord call. +type RepoPutRecord_Output struct { + Cid string `json:"cid" cborgen:"cid"` + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + +// RepoPutRecord calls the XRPC method "com.atproto.repo.putRecord". +func RepoPutRecord(ctx context.Context, c util.LexClient, input *RepoPutRecord_Input) (*RepoPutRecord_Output, error) { + var out RepoPutRecord_Output + if err := c.LexDo(ctx, util.Procedure, "application/json", "com.atproto.repo.putRecord", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto.go b/api/atproto.go deleted file mode 100644 index 2b5aea9dd..000000000 --- a/api/atproto.go +++ /dev/null @@ -1,178 +0,0 @@ -package api - -import ( - "bytes" - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -type ATProto struct { - C *xrpc.Client -} - -type TID string - -type CreateSessionResp struct { - AccessJwt string `json:"accessJwt"` - RefreshJwt string `json:"refreshJwt"` - Handle string `json:"handle"` - Did string `json:"did"` -} - -const ( - encJson = "application/json" -) - -func (atp *ATProto) CreateSession(ctx context.Context, handle, password string) (*CreateSessionResp, error) { - body := map[string]string{ - "handle": handle, - "password": password, - } - - var resp CreateSessionResp - if err := atp.C.Do(ctx, xrpc.Procedure, encJson, "com.atproto.session.create", nil, body, &resp); err != nil { - return nil, err - } - - return &resp, nil -} - -type CreateAccountResp struct { - AccessJwt string `json:"accessJwt"` - RefreshJwt string `json:"refreshJwt"` - Handle string `json:"handle"` - Did string `json:"did"` - DeclarationCid string `json:"declarationCid"` -} - -func (atp *ATProto) CreateAccount(ctx context.Context, email, handle, password string, invite *string) (*CreateAccountResp, error) { - body := map[string]string{ - "email": email, - "handle": handle, - "password": password, - } - - if invite != nil { - body["inviteCode"] = *invite - } - - var resp CreateAccountResp - if err := atp.C.Do(ctx, xrpc.Procedure, encJson, "com.atproto.account.create", nil, body, &resp); err != nil { - return nil, err - } - - return &resp, nil -} - -type CreateRecordResponse struct { - Uri string `json:"uri"` - Cid string `json:"cid"` -} - -func (atp *ATProto) RepoCreateRecord(ctx context.Context, did, collection string, validate bool, rec interface{}) (*CreateRecordResponse, error) { - body := map[string]interface{}{ - "did": did, - "collection": collection, - "validate": validate, - "record": rec, - } - - var out CreateRecordResponse - if err := atp.C.Do(ctx, xrpc.Procedure, encJson, "com.atproto.repo.createRecord", nil, body, &out); err != nil { - return nil, err - } - - return &out, nil -} - -func (atp *ATProto) SyncGetRepo(ctx context.Context, did string, from *string) ([]byte, error) { - params := map[string]interface{}{ - "did": did, - } - if from != nil { - params["from"] = *from - } - - out := new(bytes.Buffer) - if err := atp.C.Do(ctx, xrpc.Query, encJson, "com.atproto.sync.getRepo", params, nil, out); err != nil { - return nil, err - } - - return out.Bytes(), nil -} - -func (atp *ATProto) SyncGetRoot(ctx context.Context, did string) (string, error) { - params := map[string]interface{}{ - "did": did, - } - - var out struct { - Root string `json:"root"` - } - if err := atp.C.Do(ctx, xrpc.Query, encJson, "com.atproto.sync.getRoot", params, nil, &out); err != nil { - return "", err - } - - return out.Root, nil -} - -func (atp *ATProto) HandleResolve(ctx context.Context, handle string) (string, error) { - params := map[string]interface{}{ - "handle": handle, - } - - var out struct { - Did string `json:"did"` - } - if err := atp.C.Do(ctx, xrpc.Query, encJson, "com.atproto.handle.resolve", params, nil, &out); err != nil { - return "", err - } - - return out.Did, nil - -} - -func (atp *ATProto) SessionRefresh(ctx context.Context) (*xrpc.AuthInfo, error) { - var out xrpc.AuthInfo - if err := atp.C.Do(ctx, xrpc.Procedure, encJson, "com.atproto.session.refresh", nil, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} - -type RecordResponse[T any] struct { - Uri string `json:"uri"` - Cid string `json:"cid"` - Value T `json:"value"` -} - -func RepoGetRecord[T any](atp *ATProto, ctx context.Context, user string, collection string, rkey string) (*RecordResponse[T], error) { - params := map[string]interface{}{ - "user": user, - "collection": collection, - "rkey": rkey, - } - - var out RecordResponse[T] - if err := atp.C.Do(ctx, xrpc.Query, encJson, "com.atproto.repo.getRecord", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} - -func (atp *ATProto) RepoDeleteRecord(ctx context.Context, did, collection, rkey string) error { - body := map[string]interface{}{ - "did": did, - "collection": collection, - "rkey": rkey, - } - - if err := atp.C.Do(ctx, xrpc.Procedure, encJson, "com.atproto.repo.deleteRecord", nil, body, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/accountcreate.go b/api/atproto/accountcreate.go deleted file mode 100644 index cd78a2992..000000000 --- a/api/atproto/accountcreate.go +++ /dev/null @@ -1,38 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.account.create - -func init() { -} - -type AccountCreate_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Email string `json:"email" cborgen:"email"` - Handle string `json:"handle" cborgen:"handle"` - InviteCode *string `json:"inviteCode,omitempty" cborgen:"inviteCode"` - Password string `json:"password" cborgen:"password"` - RecoveryKey *string `json:"recoveryKey,omitempty" cborgen:"recoveryKey"` -} - -type AccountCreate_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` - Did string `json:"did" cborgen:"did"` - Handle string `json:"handle" cborgen:"handle"` - RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` -} - -func AccountCreate(ctx context.Context, c *xrpc.Client, input *AccountCreate_Input) (*AccountCreate_Output, error) { - var out AccountCreate_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.account.create", nil, input, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/accountcreateInviteCode.go b/api/atproto/accountcreateInviteCode.go deleted file mode 100644 index 99b69653c..000000000 --- a/api/atproto/accountcreateInviteCode.go +++ /dev/null @@ -1,31 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.account.createInviteCode - -func init() { -} - -type AccountCreateInviteCode_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - UseCount int64 `json:"useCount" cborgen:"useCount"` -} - -type AccountCreateInviteCode_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Code string `json:"code" cborgen:"code"` -} - -func AccountCreateInviteCode(ctx context.Context, c *xrpc.Client, input *AccountCreateInviteCode_Input) (*AccountCreateInviteCode_Output, error) { - var out AccountCreateInviteCode_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.account.createInviteCode", nil, input, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/accountdelete.go b/api/atproto/accountdelete.go deleted file mode 100644 index 98a0884b3..000000000 --- a/api/atproto/accountdelete.go +++ /dev/null @@ -1,19 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.account.delete - -func init() { -} -func AccountDelete(ctx context.Context, c *xrpc.Client) error { - if err := c.Do(ctx, xrpc.Procedure, "", "com.atproto.account.delete", nil, nil, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/accountget.go b/api/atproto/accountget.go deleted file mode 100644 index a89fd4bc3..000000000 --- a/api/atproto/accountget.go +++ /dev/null @@ -1,19 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.account.get - -func init() { -} -func AccountGet(ctx context.Context, c *xrpc.Client) error { - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.account.get", nil, nil, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/accountrequestPasswordReset.go b/api/atproto/accountrequestPasswordReset.go deleted file mode 100644 index b295115e2..000000000 --- a/api/atproto/accountrequestPasswordReset.go +++ /dev/null @@ -1,25 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.account.requestPasswordReset - -func init() { -} - -type AccountRequestPasswordReset_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Email string `json:"email" cborgen:"email"` -} - -func AccountRequestPasswordReset(ctx context.Context, c *xrpc.Client, input *AccountRequestPasswordReset_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.account.requestPasswordReset", nil, input, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/accountresetPassword.go b/api/atproto/accountresetPassword.go deleted file mode 100644 index 59da7f269..000000000 --- a/api/atproto/accountresetPassword.go +++ /dev/null @@ -1,26 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.account.resetPassword - -func init() { -} - -type AccountResetPassword_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Password string `json:"password" cborgen:"password"` - Token string `json:"token" cborgen:"token"` -} - -func AccountResetPassword(ctx context.Context, c *xrpc.Client, input *AccountResetPassword_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.account.resetPassword", nil, input, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/admindefs.go b/api/atproto/admindefs.go new file mode 100644 index 000000000..b298e958c --- /dev/null +++ b/api/atproto/admindefs.go @@ -0,0 +1,51 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.defs + +package atproto + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminDefs_AccountView is a "accountView" in the com.atproto.admin.defs schema. +type AdminDefs_AccountView struct { + DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` + Did string `json:"did" cborgen:"did"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailConfirmedAt *string `json:"emailConfirmedAt,omitempty" cborgen:"emailConfirmedAt,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` + InvitedBy *ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` + Invites []*ServerDefs_InviteCode `json:"invites,omitempty" cborgen:"invites,omitempty"` + InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` + RelatedRecords []*lexutil.LexiconTypeDecoder `json:"relatedRecords,omitempty" cborgen:"relatedRecords,omitempty"` + ThreatSignatures []*AdminDefs_ThreatSignature `json:"threatSignatures,omitempty" cborgen:"threatSignatures,omitempty"` +} + +// AdminDefs_RepoBlobRef is a "repoBlobRef" in the com.atproto.admin.defs schema. +type AdminDefs_RepoBlobRef struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.admin.defs#repoBlobRef"` + Cid string `json:"cid" cborgen:"cid"` + Did string `json:"did" cborgen:"did"` + RecordUri *string `json:"recordUri,omitempty" cborgen:"recordUri,omitempty"` +} + +// AdminDefs_RepoRef is a "repoRef" in the com.atproto.admin.defs schema. +type AdminDefs_RepoRef struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.admin.defs#repoRef"` + Did string `json:"did" cborgen:"did"` +} + +// AdminDefs_StatusAttr is a "statusAttr" in the com.atproto.admin.defs schema. +type AdminDefs_StatusAttr struct { + Applied bool `json:"applied" cborgen:"applied"` + Ref *string `json:"ref,omitempty" cborgen:"ref,omitempty"` +} + +// AdminDefs_ThreatSignature is a "threatSignature" in the com.atproto.admin.defs schema. +type AdminDefs_ThreatSignature struct { + Property string `json:"property" cborgen:"property"` + Value string `json:"value" cborgen:"value"` +} diff --git a/api/atproto/admindeleteAccount.go b/api/atproto/admindeleteAccount.go new file mode 100644 index 000000000..73cd28a7f --- /dev/null +++ b/api/atproto/admindeleteAccount.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.deleteAccount + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminDeleteAccount_Input is the input argument to a com.atproto.admin.deleteAccount call. +type AdminDeleteAccount_Input struct { + Did string `json:"did" cborgen:"did"` +} + +// AdminDeleteAccount calls the XRPC method "com.atproto.admin.deleteAccount". +func AdminDeleteAccount(ctx context.Context, c lexutil.LexClient, input *AdminDeleteAccount_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.deleteAccount", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/admindisableAccountInvites.go b/api/atproto/admindisableAccountInvites.go new file mode 100644 index 000000000..4881d41fa --- /dev/null +++ b/api/atproto/admindisableAccountInvites.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.disableAccountInvites + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminDisableAccountInvites_Input is the input argument to a com.atproto.admin.disableAccountInvites call. +type AdminDisableAccountInvites_Input struct { + Account string `json:"account" cborgen:"account"` + // note: Optional reason for disabled invites. + Note *string `json:"note,omitempty" cborgen:"note,omitempty"` +} + +// AdminDisableAccountInvites calls the XRPC method "com.atproto.admin.disableAccountInvites". +func AdminDisableAccountInvites(ctx context.Context, c lexutil.LexClient, input *AdminDisableAccountInvites_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.disableAccountInvites", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/admindisableInviteCodes.go b/api/atproto/admindisableInviteCodes.go new file mode 100644 index 000000000..4eaf5996f --- /dev/null +++ b/api/atproto/admindisableInviteCodes.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.disableInviteCodes + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminDisableInviteCodes_Input is the input argument to a com.atproto.admin.disableInviteCodes call. +type AdminDisableInviteCodes_Input struct { + Accounts []string `json:"accounts,omitempty" cborgen:"accounts,omitempty"` + Codes []string `json:"codes,omitempty" cborgen:"codes,omitempty"` +} + +// AdminDisableInviteCodes calls the XRPC method "com.atproto.admin.disableInviteCodes". +func AdminDisableInviteCodes(ctx context.Context, c lexutil.LexClient, input *AdminDisableInviteCodes_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.disableInviteCodes", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/adminenableAccountInvites.go b/api/atproto/adminenableAccountInvites.go new file mode 100644 index 000000000..9aef09835 --- /dev/null +++ b/api/atproto/adminenableAccountInvites.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.enableAccountInvites + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminEnableAccountInvites_Input is the input argument to a com.atproto.admin.enableAccountInvites call. +type AdminEnableAccountInvites_Input struct { + Account string `json:"account" cborgen:"account"` + // note: Optional reason for enabled invites. + Note *string `json:"note,omitempty" cborgen:"note,omitempty"` +} + +// AdminEnableAccountInvites calls the XRPC method "com.atproto.admin.enableAccountInvites". +func AdminEnableAccountInvites(ctx context.Context, c lexutil.LexClient, input *AdminEnableAccountInvites_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.enableAccountInvites", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/admingetAccountInfo.go b/api/atproto/admingetAccountInfo.go new file mode 100644 index 000000000..de3ccb242 --- /dev/null +++ b/api/atproto/admingetAccountInfo.go @@ -0,0 +1,24 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.getAccountInfo + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminGetAccountInfo calls the XRPC method "com.atproto.admin.getAccountInfo". +func AdminGetAccountInfo(ctx context.Context, c lexutil.LexClient, did string) (*AdminDefs_AccountView, error) { + var out AdminDefs_AccountView + + params := map[string]interface{}{} + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.admin.getAccountInfo", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/admingetAccountInfos.go b/api/atproto/admingetAccountInfos.go new file mode 100644 index 000000000..93318f9e6 --- /dev/null +++ b/api/atproto/admingetAccountInfos.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.getAccountInfos + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminGetAccountInfos_Output is the output of a com.atproto.admin.getAccountInfos call. +type AdminGetAccountInfos_Output struct { + Infos []*AdminDefs_AccountView `json:"infos" cborgen:"infos"` +} + +// AdminGetAccountInfos calls the XRPC method "com.atproto.admin.getAccountInfos". +func AdminGetAccountInfos(ctx context.Context, c lexutil.LexClient, dids []string) (*AdminGetAccountInfos_Output, error) { + var out AdminGetAccountInfos_Output + + params := map[string]interface{}{} + params["dids"] = dids + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.admin.getAccountInfos", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/admingetInviteCodes.go b/api/atproto/admingetInviteCodes.go new file mode 100644 index 000000000..d3e15a7c4 --- /dev/null +++ b/api/atproto/admingetInviteCodes.go @@ -0,0 +1,38 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.getInviteCodes + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminGetInviteCodes_Output is the output of a com.atproto.admin.getInviteCodes call. +type AdminGetInviteCodes_Output struct { + Codes []*ServerDefs_InviteCode `json:"codes" cborgen:"codes"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// AdminGetInviteCodes calls the XRPC method "com.atproto.admin.getInviteCodes". +func AdminGetInviteCodes(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, sort string) (*AdminGetInviteCodes_Output, error) { + var out AdminGetInviteCodes_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if sort != "" { + params["sort"] = sort + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.admin.getInviteCodes", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/admingetSubjectStatus.go b/api/atproto/admingetSubjectStatus.go new file mode 100644 index 000000000..29c4adcb9 --- /dev/null +++ b/api/atproto/admingetSubjectStatus.go @@ -0,0 +1,84 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.getSubjectStatus + +package atproto + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminGetSubjectStatus_Output is the output of a com.atproto.admin.getSubjectStatus call. +type AdminGetSubjectStatus_Output struct { + Deactivated *AdminDefs_StatusAttr `json:"deactivated,omitempty" cborgen:"deactivated,omitempty"` + Subject *AdminGetSubjectStatus_Output_Subject `json:"subject" cborgen:"subject"` + Takedown *AdminDefs_StatusAttr `json:"takedown,omitempty" cborgen:"takedown,omitempty"` +} + +type AdminGetSubjectStatus_Output_Subject struct { + AdminDefs_RepoRef *AdminDefs_RepoRef + RepoStrongRef *RepoStrongRef + AdminDefs_RepoBlobRef *AdminDefs_RepoBlobRef +} + +func (t *AdminGetSubjectStatus_Output_Subject) MarshalJSON() ([]byte, error) { + if t.AdminDefs_RepoRef != nil { + t.AdminDefs_RepoRef.LexiconTypeID = "com.atproto.admin.defs#repoRef" + return json.Marshal(t.AdminDefs_RepoRef) + } + if t.RepoStrongRef != nil { + t.RepoStrongRef.LexiconTypeID = "com.atproto.repo.strongRef" + return json.Marshal(t.RepoStrongRef) + } + if t.AdminDefs_RepoBlobRef != nil { + t.AdminDefs_RepoBlobRef.LexiconTypeID = "com.atproto.admin.defs#repoBlobRef" + return json.Marshal(t.AdminDefs_RepoBlobRef) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *AdminGetSubjectStatus_Output_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.admin.defs#repoRef": + t.AdminDefs_RepoRef = new(AdminDefs_RepoRef) + return json.Unmarshal(b, t.AdminDefs_RepoRef) + case "com.atproto.repo.strongRef": + t.RepoStrongRef = new(RepoStrongRef) + return json.Unmarshal(b, t.RepoStrongRef) + case "com.atproto.admin.defs#repoBlobRef": + t.AdminDefs_RepoBlobRef = new(AdminDefs_RepoBlobRef) + return json.Unmarshal(b, t.AdminDefs_RepoBlobRef) + default: + return nil + } +} + +// AdminGetSubjectStatus calls the XRPC method "com.atproto.admin.getSubjectStatus". +func AdminGetSubjectStatus(ctx context.Context, c lexutil.LexClient, blob string, did string, uri string) (*AdminGetSubjectStatus_Output, error) { + var out AdminGetSubjectStatus_Output + + params := map[string]interface{}{} + if blob != "" { + params["blob"] = blob + } + if did != "" { + params["did"] = did + } + if uri != "" { + params["uri"] = uri + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.admin.getSubjectStatus", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/adminsearchAccounts.go b/api/atproto/adminsearchAccounts.go new file mode 100644 index 000000000..fa43fb1ec --- /dev/null +++ b/api/atproto/adminsearchAccounts.go @@ -0,0 +1,38 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.searchAccounts + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminSearchAccounts_Output is the output of a com.atproto.admin.searchAccounts call. +type AdminSearchAccounts_Output struct { + Accounts []*AdminDefs_AccountView `json:"accounts" cborgen:"accounts"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// AdminSearchAccounts calls the XRPC method "com.atproto.admin.searchAccounts". +func AdminSearchAccounts(ctx context.Context, c lexutil.LexClient, cursor string, email string, limit int64) (*AdminSearchAccounts_Output, error) { + var out AdminSearchAccounts_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if email != "" { + params["email"] = email + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.admin.searchAccounts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/adminsendEmail.go b/api/atproto/adminsendEmail.go new file mode 100644 index 000000000..0dacd5e4a --- /dev/null +++ b/api/atproto/adminsendEmail.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.sendEmail + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminSendEmail_Input is the input argument to a com.atproto.admin.sendEmail call. +type AdminSendEmail_Input struct { + // comment: Additional comment by the sender that won't be used in the email itself but helpful to provide more context for moderators/reviewers + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + Content string `json:"content" cborgen:"content"` + RecipientDid string `json:"recipientDid" cborgen:"recipientDid"` + SenderDid string `json:"senderDid" cborgen:"senderDid"` + Subject *string `json:"subject,omitempty" cborgen:"subject,omitempty"` +} + +// AdminSendEmail_Output is the output of a com.atproto.admin.sendEmail call. +type AdminSendEmail_Output struct { + Sent bool `json:"sent" cborgen:"sent"` +} + +// AdminSendEmail calls the XRPC method "com.atproto.admin.sendEmail". +func AdminSendEmail(ctx context.Context, c lexutil.LexClient, input *AdminSendEmail_Input) (*AdminSendEmail_Output, error) { + var out AdminSendEmail_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.sendEmail", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/adminupdateAccountEmail.go b/api/atproto/adminupdateAccountEmail.go new file mode 100644 index 000000000..ab108f1a4 --- /dev/null +++ b/api/atproto/adminupdateAccountEmail.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.updateAccountEmail + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminUpdateAccountEmail_Input is the input argument to a com.atproto.admin.updateAccountEmail call. +type AdminUpdateAccountEmail_Input struct { + // account: The handle or DID of the repo. + Account string `json:"account" cborgen:"account"` + Email string `json:"email" cborgen:"email"` +} + +// AdminUpdateAccountEmail calls the XRPC method "com.atproto.admin.updateAccountEmail". +func AdminUpdateAccountEmail(ctx context.Context, c lexutil.LexClient, input *AdminUpdateAccountEmail_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.updateAccountEmail", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/adminupdateAccountHandle.go b/api/atproto/adminupdateAccountHandle.go new file mode 100644 index 000000000..46261d95f --- /dev/null +++ b/api/atproto/adminupdateAccountHandle.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.updateAccountHandle + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminUpdateAccountHandle_Input is the input argument to a com.atproto.admin.updateAccountHandle call. +type AdminUpdateAccountHandle_Input struct { + Did string `json:"did" cborgen:"did"` + Handle string `json:"handle" cborgen:"handle"` +} + +// AdminUpdateAccountHandle calls the XRPC method "com.atproto.admin.updateAccountHandle". +func AdminUpdateAccountHandle(ctx context.Context, c lexutil.LexClient, input *AdminUpdateAccountHandle_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.updateAccountHandle", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/adminupdateAccountPassword.go b/api/atproto/adminupdateAccountPassword.go new file mode 100644 index 000000000..446719af1 --- /dev/null +++ b/api/atproto/adminupdateAccountPassword.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.updateAccountPassword + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminUpdateAccountPassword_Input is the input argument to a com.atproto.admin.updateAccountPassword call. +type AdminUpdateAccountPassword_Input struct { + Did string `json:"did" cborgen:"did"` + Password string `json:"password" cborgen:"password"` +} + +// AdminUpdateAccountPassword calls the XRPC method "com.atproto.admin.updateAccountPassword". +func AdminUpdateAccountPassword(ctx context.Context, c lexutil.LexClient, input *AdminUpdateAccountPassword_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.updateAccountPassword", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/adminupdateAccountSigningKey.go b/api/atproto/adminupdateAccountSigningKey.go new file mode 100644 index 000000000..cd73d2879 --- /dev/null +++ b/api/atproto/adminupdateAccountSigningKey.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.updateAccountSigningKey + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminUpdateAccountSigningKey_Input is the input argument to a com.atproto.admin.updateAccountSigningKey call. +type AdminUpdateAccountSigningKey_Input struct { + Did string `json:"did" cborgen:"did"` + // signingKey: Did-key formatted public key + SigningKey string `json:"signingKey" cborgen:"signingKey"` +} + +// AdminUpdateAccountSigningKey calls the XRPC method "com.atproto.admin.updateAccountSigningKey". +func AdminUpdateAccountSigningKey(ctx context.Context, c lexutil.LexClient, input *AdminUpdateAccountSigningKey_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.updateAccountSigningKey", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/adminupdateSubjectStatus.go b/api/atproto/adminupdateSubjectStatus.go new file mode 100644 index 000000000..610c8ac6e --- /dev/null +++ b/api/atproto/adminupdateSubjectStatus.go @@ -0,0 +1,122 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.admin.updateSubjectStatus + +package atproto + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AdminUpdateSubjectStatus_Input is the input argument to a com.atproto.admin.updateSubjectStatus call. +type AdminUpdateSubjectStatus_Input struct { + Deactivated *AdminDefs_StatusAttr `json:"deactivated,omitempty" cborgen:"deactivated,omitempty"` + Subject *AdminUpdateSubjectStatus_Input_Subject `json:"subject" cborgen:"subject"` + Takedown *AdminDefs_StatusAttr `json:"takedown,omitempty" cborgen:"takedown,omitempty"` +} + +type AdminUpdateSubjectStatus_Input_Subject struct { + AdminDefs_RepoRef *AdminDefs_RepoRef + RepoStrongRef *RepoStrongRef + AdminDefs_RepoBlobRef *AdminDefs_RepoBlobRef +} + +func (t *AdminUpdateSubjectStatus_Input_Subject) MarshalJSON() ([]byte, error) { + if t.AdminDefs_RepoRef != nil { + t.AdminDefs_RepoRef.LexiconTypeID = "com.atproto.admin.defs#repoRef" + return json.Marshal(t.AdminDefs_RepoRef) + } + if t.RepoStrongRef != nil { + t.RepoStrongRef.LexiconTypeID = "com.atproto.repo.strongRef" + return json.Marshal(t.RepoStrongRef) + } + if t.AdminDefs_RepoBlobRef != nil { + t.AdminDefs_RepoBlobRef.LexiconTypeID = "com.atproto.admin.defs#repoBlobRef" + return json.Marshal(t.AdminDefs_RepoBlobRef) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *AdminUpdateSubjectStatus_Input_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.admin.defs#repoRef": + t.AdminDefs_RepoRef = new(AdminDefs_RepoRef) + return json.Unmarshal(b, t.AdminDefs_RepoRef) + case "com.atproto.repo.strongRef": + t.RepoStrongRef = new(RepoStrongRef) + return json.Unmarshal(b, t.RepoStrongRef) + case "com.atproto.admin.defs#repoBlobRef": + t.AdminDefs_RepoBlobRef = new(AdminDefs_RepoBlobRef) + return json.Unmarshal(b, t.AdminDefs_RepoBlobRef) + default: + return nil + } +} + +// AdminUpdateSubjectStatus_Output is the output of a com.atproto.admin.updateSubjectStatus call. +type AdminUpdateSubjectStatus_Output struct { + Subject *AdminUpdateSubjectStatus_Output_Subject `json:"subject" cborgen:"subject"` + Takedown *AdminDefs_StatusAttr `json:"takedown,omitempty" cborgen:"takedown,omitempty"` +} + +type AdminUpdateSubjectStatus_Output_Subject struct { + AdminDefs_RepoRef *AdminDefs_RepoRef + RepoStrongRef *RepoStrongRef + AdminDefs_RepoBlobRef *AdminDefs_RepoBlobRef +} + +func (t *AdminUpdateSubjectStatus_Output_Subject) MarshalJSON() ([]byte, error) { + if t.AdminDefs_RepoRef != nil { + t.AdminDefs_RepoRef.LexiconTypeID = "com.atproto.admin.defs#repoRef" + return json.Marshal(t.AdminDefs_RepoRef) + } + if t.RepoStrongRef != nil { + t.RepoStrongRef.LexiconTypeID = "com.atproto.repo.strongRef" + return json.Marshal(t.RepoStrongRef) + } + if t.AdminDefs_RepoBlobRef != nil { + t.AdminDefs_RepoBlobRef.LexiconTypeID = "com.atproto.admin.defs#repoBlobRef" + return json.Marshal(t.AdminDefs_RepoBlobRef) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *AdminUpdateSubjectStatus_Output_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.admin.defs#repoRef": + t.AdminDefs_RepoRef = new(AdminDefs_RepoRef) + return json.Unmarshal(b, t.AdminDefs_RepoRef) + case "com.atproto.repo.strongRef": + t.RepoStrongRef = new(RepoStrongRef) + return json.Unmarshal(b, t.RepoStrongRef) + case "com.atproto.admin.defs#repoBlobRef": + t.AdminDefs_RepoBlobRef = new(AdminDefs_RepoBlobRef) + return json.Unmarshal(b, t.AdminDefs_RepoBlobRef) + default: + return nil + } +} + +// AdminUpdateSubjectStatus calls the XRPC method "com.atproto.admin.updateSubjectStatus". +func AdminUpdateSubjectStatus(ctx context.Context, c lexutil.LexClient, input *AdminUpdateSubjectStatus_Input) (*AdminUpdateSubjectStatus_Output, error) { + var out AdminUpdateSubjectStatus_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.admin.updateSubjectStatus", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/blobupload.go b/api/atproto/blobupload.go deleted file mode 100644 index b290136d1..000000000 --- a/api/atproto/blobupload.go +++ /dev/null @@ -1,27 +0,0 @@ -package schemagen - -import ( - "context" - "io" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.blob.upload - -func init() { -} - -type BlobUpload_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid string `json:"cid" cborgen:"cid"` -} - -func BlobUpload(ctx context.Context, c *xrpc.Client, input io.Reader) (*BlobUpload_Output, error) { - var out BlobUpload_Output - if err := c.Do(ctx, xrpc.Procedure, "*/*", "com.atproto.blob.upload", nil, input, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/cbor_gen.go b/api/atproto/cbor_gen.go index 82eb7729f..8f2811b79 100644 --- a/api/atproto/cbor_gen.go +++ b/api/atproto/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package schemagen +package atproto import ( "fmt" @@ -8,6 +8,7 @@ import ( "math" "sort" + util "github.com/bluesky-social/indigo/lex/util" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" @@ -18,6 +19,151 @@ var _ = cid.Undef var _ = math.E var _ = sort.Sort +func (t *LexiconSchema) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("com.atproto.lexicon.schema"))); err != nil { + return err + } + if _, err := cw.WriteString(string("com.atproto.lexicon.schema")); err != nil { + return err + } + + // t.Lexicon (int64) (int64) + if len("lexicon") > 1000000 { + return xerrors.Errorf("Value in field \"lexicon\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("lexicon"))); err != nil { + return err + } + if _, err := cw.WriteString(string("lexicon")); err != nil { + return err + } + + if t.Lexicon >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Lexicon)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Lexicon-1)); err != nil { + return err + } + } + + return nil +} + +func (t *LexiconSchema) UnmarshalCBOR(r io.Reader) (err error) { + *t = LexiconSchema{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LexiconSchema: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 7) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Lexicon (int64) (int64) + case "lexicon": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Lexicon = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} func (t *RepoStrongRef) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -25,78 +171,82 @@ func (t *RepoStrongRef) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) + fieldCount := 3 - if _, err := cw.Write([]byte{163}); err != nil { + if t.LexiconTypeID == "" { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } // t.Cid (string) (string) - if len("cid") > cbg.MaxLength { + if len("cid") > 1000000 { return xerrors.Errorf("Value in field \"cid\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil { return err } - if _, err := io.WriteString(w, string("cid")); err != nil { + if _, err := cw.WriteString(string("cid")); err != nil { return err } - if len(t.Cid) > cbg.MaxLength { + if len(t.Cid) > 1000000 { return xerrors.Errorf("Value in field t.Cid was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Cid))); err != nil { return err } - if _, err := io.WriteString(w, string(t.Cid)); err != nil { + if _, err := cw.WriteString(string(t.Cid)); err != nil { return err } // t.Uri (string) (string) - if len("uri") > cbg.MaxLength { + if len("uri") > 1000000 { return xerrors.Errorf("Value in field \"uri\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { return err } - if _, err := io.WriteString(w, string("uri")); err != nil { + if _, err := cw.WriteString(string("uri")); err != nil { return err } - if len(t.Uri) > cbg.MaxLength { + if len(t.Uri) > 1000000 { return xerrors.Errorf("Value in field t.Uri was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { return err } - if _, err := io.WriteString(w, string(t.Uri)); err != nil { + if _, err := cw.WriteString(string(t.Uri)); err != nil { return err } // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") - } + if t.LexiconTypeID != "" { - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { - return err - } + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { - return err + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("com.atproto.repo.strongRef"))); err != nil { + return err + } + if _, err := cw.WriteString(string("com.atproto.repo.strongRef")); err != nil { + return err + } } return nil } @@ -124,26 +274,29 @@ func (t *RepoStrongRef) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("RepoStrongRef: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Cid (string) (string) case "cid": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -154,7 +307,7 @@ func (t *RepoStrongRef) UnmarshalCBOR(r io.Reader) (err error) { case "uri": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -162,10 +315,10 @@ func (t *RepoStrongRef) UnmarshalCBOR(r io.Reader) (err error) { t.Uri = string(sval) } // t.LexiconTypeID (string) (string) - case "LexiconTypeID": + case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -175,7 +328,3440 @@ func (t *RepoStrongRef) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *SyncSubscribeRepos_Commit) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 12 + + if t.Blocks == nil { + fieldCount-- + } + + if t.PrevData == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Ops ([]*atproto.SyncSubscribeRepos_RepoOp) (slice) + if len("ops") > 1000000 { + return xerrors.Errorf("Value in field \"ops\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ops"))); err != nil { + return err + } + if _, err := cw.WriteString(string("ops")); err != nil { + return err + } + + if len(t.Ops) > 8192 { + return xerrors.Errorf("Slice value in field t.Ops was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Ops))); err != nil { + return err + } + for _, v := range t.Ops { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + + // t.Rev (string) (string) + if len("rev") > 1000000 { + return xerrors.Errorf("Value in field \"rev\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("rev"))); err != nil { + return err + } + if _, err := cw.WriteString(string("rev")); err != nil { + return err + } + + if len(t.Rev) > 1000000 { + return xerrors.Errorf("Value in field t.Rev was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Rev))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Rev)); err != nil { + return err + } + + // t.Seq (int64) (int64) + if len("seq") > 1000000 { + return xerrors.Errorf("Value in field \"seq\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("seq"))); err != nil { + return err + } + if _, err := cw.WriteString(string("seq")); err != nil { + return err + } + + if t.Seq >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Seq)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Seq-1)); err != nil { + return err + } + } + + // t.Repo (string) (string) + if len("repo") > 1000000 { + return xerrors.Errorf("Value in field \"repo\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repo"))); err != nil { + return err + } + if _, err := cw.WriteString(string("repo")); err != nil { + return err + } + + if len(t.Repo) > 1000000 { + return xerrors.Errorf("Value in field t.Repo was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Repo))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Repo)); err != nil { + return err + } + + // t.Time (string) (string) + if len("time") > 1000000 { + return xerrors.Errorf("Value in field \"time\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("time"))); err != nil { + return err + } + if _, err := cw.WriteString(string("time")); err != nil { + return err + } + + if len(t.Time) > 1000000 { + return xerrors.Errorf("Value in field t.Time was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Time))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Time)); err != nil { + return err + } + + // t.Blobs ([]util.LexLink) (slice) + if len("blobs") > 1000000 { + return xerrors.Errorf("Value in field \"blobs\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blobs"))); err != nil { + return err + } + if _, err := cw.WriteString(string("blobs")); err != nil { + return err + } + + if len(t.Blobs) > 8192 { + return xerrors.Errorf("Slice value in field t.Blobs was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Blobs))); err != nil { + return err + } + for _, v := range t.Blobs { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + + // t.Since (string) (string) + if len("since") > 1000000 { + return xerrors.Errorf("Value in field \"since\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("since"))); err != nil { + return err + } + if _, err := cw.WriteString(string("since")); err != nil { + return err + } + + if t.Since == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Since) > 1000000 { + return xerrors.Errorf("Value in field t.Since was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Since))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Since)); err != nil { + return err + } + } + + // t.Blocks (util.LexBytes) (slice) + if t.Blocks != nil { + + if len("blocks") > 1000000 { + return xerrors.Errorf("Value in field \"blocks\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blocks"))); err != nil { + return err + } + if _, err := cw.WriteString(string("blocks")); err != nil { + return err + } + + if len(t.Blocks) > 2097152 { + return xerrors.Errorf("Byte array in field t.Blocks was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Blocks))); err != nil { + return err + } + + if _, err := cw.Write(t.Blocks); err != nil { + return err + } + + } + + // t.Commit (util.LexLink) (struct) + if len("commit") > 1000000 { + return xerrors.Errorf("Value in field \"commit\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("commit"))); err != nil { + return err + } + if _, err := cw.WriteString(string("commit")); err != nil { + return err + } + + if err := t.Commit.MarshalCBOR(cw); err != nil { + return err + } + + // t.Rebase (bool) (bool) + if len("rebase") > 1000000 { + return xerrors.Errorf("Value in field \"rebase\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("rebase"))); err != nil { + return err + } + if _, err := cw.WriteString(string("rebase")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.Rebase); err != nil { + return err + } + + // t.TooBig (bool) (bool) + if len("tooBig") > 1000000 { + return xerrors.Errorf("Value in field \"tooBig\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("tooBig"))); err != nil { + return err + } + if _, err := cw.WriteString(string("tooBig")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.TooBig); err != nil { + return err + } + + // t.PrevData (util.LexLink) (struct) + if t.PrevData != nil { + + if len("prevData") > 1000000 { + return xerrors.Errorf("Value in field \"prevData\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("prevData"))); err != nil { + return err + } + if _, err := cw.WriteString(string("prevData")); err != nil { + return err + } + + if err := t.PrevData.MarshalCBOR(cw); err != nil { + return err + } + } + return nil +} + +func (t *SyncSubscribeRepos_Commit) UnmarshalCBOR(r io.Reader) (err error) { + *t = SyncSubscribeRepos_Commit{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SyncSubscribeRepos_Commit: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 8) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Ops ([]*atproto.SyncSubscribeRepos_RepoOp) (slice) + case "ops": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Ops: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Ops = make([]*SyncSubscribeRepos_RepoOp, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Ops[i] = new(SyncSubscribeRepos_RepoOp) + if err := t.Ops[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Ops[i] pointer: %w", err) + } + } + + } + + } + } + // t.Rev (string) (string) + case "rev": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Rev = string(sval) + } + // t.Seq (int64) (int64) + case "seq": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Seq = int64(extraI) + } + // t.Repo (string) (string) + case "repo": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Repo = string(sval) + } + // t.Time (string) (string) + case "time": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Time = string(sval) + } + // t.Blobs ([]util.LexLink) (slice) + case "blobs": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Blobs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Blobs = make([]util.LexLink, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + if err := t.Blobs[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Blobs[i]: %w", err) + } + + } + + } + } + // t.Since (string) (string) + case "since": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Since = (*string)(&sval) + } + } + // t.Blocks (util.LexBytes) (slice) + case "blocks": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.Blocks: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Blocks = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Blocks); err != nil { + return err + } + + // t.Commit (util.LexLink) (struct) + case "commit": + + { + + if err := t.Commit.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Commit: %w", err) + } + + } + // t.Rebase (bool) (bool) + case "rebase": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.Rebase = false + case 21: + t.Rebase = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.TooBig (bool) (bool) + case "tooBig": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.TooBig = false + case 21: + t.TooBig = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.PrevData (util.LexLink) (struct) + case "prevData": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.PrevData = new(util.LexLink) + if err := t.PrevData.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.PrevData pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *SyncSubscribeRepos_Sync) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 5 + + if t.Blocks == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Did (string) (string) + if len("did") > 1000000 { + return xerrors.Errorf("Value in field \"did\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil { + return err + } + if _, err := cw.WriteString(string("did")); err != nil { + return err + } + + if len(t.Did) > 1000000 { + return xerrors.Errorf("Value in field t.Did was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Did))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Did)); err != nil { + return err + } + + // t.Rev (string) (string) + if len("rev") > 1000000 { + return xerrors.Errorf("Value in field \"rev\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("rev"))); err != nil { + return err + } + if _, err := cw.WriteString(string("rev")); err != nil { + return err + } + + if len(t.Rev) > 1000000 { + return xerrors.Errorf("Value in field t.Rev was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Rev))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Rev)); err != nil { + return err + } + + // t.Seq (int64) (int64) + if len("seq") > 1000000 { + return xerrors.Errorf("Value in field \"seq\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("seq"))); err != nil { + return err + } + if _, err := cw.WriteString(string("seq")); err != nil { + return err + } + + if t.Seq >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Seq)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Seq-1)); err != nil { + return err + } + } + + // t.Time (string) (string) + if len("time") > 1000000 { + return xerrors.Errorf("Value in field \"time\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("time"))); err != nil { + return err + } + if _, err := cw.WriteString(string("time")); err != nil { + return err + } + + if len(t.Time) > 1000000 { + return xerrors.Errorf("Value in field t.Time was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Time))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Time)); err != nil { + return err + } + + // t.Blocks (util.LexBytes) (slice) + if t.Blocks != nil { + + if len("blocks") > 1000000 { + return xerrors.Errorf("Value in field \"blocks\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blocks"))); err != nil { + return err + } + if _, err := cw.WriteString(string("blocks")); err != nil { + return err + } + + if len(t.Blocks) > 2097152 { + return xerrors.Errorf("Byte array in field t.Blocks was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Blocks))); err != nil { + return err + } + + if _, err := cw.Write(t.Blocks); err != nil { + return err + } + + } + return nil +} + +func (t *SyncSubscribeRepos_Sync) UnmarshalCBOR(r io.Reader) (err error) { + *t = SyncSubscribeRepos_Sync{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SyncSubscribeRepos_Sync: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Did (string) (string) + case "did": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Did = string(sval) + } + // t.Rev (string) (string) + case "rev": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Rev = string(sval) + } + // t.Seq (int64) (int64) + case "seq": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Seq = int64(extraI) + } + // t.Time (string) (string) + case "time": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Time = string(sval) + } + // t.Blocks (util.LexBytes) (slice) + case "blocks": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.Blocks: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Blocks = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Blocks); err != nil { + return err + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *SyncSubscribeRepos_Identity) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 4 + + if t.Handle == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Did (string) (string) + if len("did") > 1000000 { + return xerrors.Errorf("Value in field \"did\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil { + return err + } + if _, err := cw.WriteString(string("did")); err != nil { + return err + } + + if len(t.Did) > 1000000 { + return xerrors.Errorf("Value in field t.Did was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Did))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Did)); err != nil { + return err + } + + // t.Seq (int64) (int64) + if len("seq") > 1000000 { + return xerrors.Errorf("Value in field \"seq\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("seq"))); err != nil { + return err + } + if _, err := cw.WriteString(string("seq")); err != nil { + return err + } + + if t.Seq >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Seq)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Seq-1)); err != nil { + return err + } + } + + // t.Time (string) (string) + if len("time") > 1000000 { + return xerrors.Errorf("Value in field \"time\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("time"))); err != nil { + return err + } + if _, err := cw.WriteString(string("time")); err != nil { + return err + } + + if len(t.Time) > 1000000 { + return xerrors.Errorf("Value in field t.Time was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Time))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Time)); err != nil { + return err + } + + // t.Handle (string) (string) + if t.Handle != nil { + + if len("handle") > 1000000 { + return xerrors.Errorf("Value in field \"handle\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("handle"))); err != nil { + return err + } + if _, err := cw.WriteString(string("handle")); err != nil { + return err + } + + if t.Handle == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Handle) > 1000000 { + return xerrors.Errorf("Value in field t.Handle was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Handle))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Handle)); err != nil { + return err + } + } + } + return nil +} + +func (t *SyncSubscribeRepos_Identity) UnmarshalCBOR(r io.Reader) (err error) { + *t = SyncSubscribeRepos_Identity{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SyncSubscribeRepos_Identity: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Did (string) (string) + case "did": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Did = string(sval) + } + // t.Seq (int64) (int64) + case "seq": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Seq = int64(extraI) + } + // t.Time (string) (string) + case "time": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Time = string(sval) + } + // t.Handle (string) (string) + case "handle": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Handle = (*string)(&sval) + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *SyncSubscribeRepos_Account) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 5 + + if t.Status == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Did (string) (string) + if len("did") > 1000000 { + return xerrors.Errorf("Value in field \"did\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil { + return err + } + if _, err := cw.WriteString(string("did")); err != nil { + return err + } + + if len(t.Did) > 1000000 { + return xerrors.Errorf("Value in field t.Did was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Did))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Did)); err != nil { + return err + } + + // t.Seq (int64) (int64) + if len("seq") > 1000000 { + return xerrors.Errorf("Value in field \"seq\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("seq"))); err != nil { + return err + } + if _, err := cw.WriteString(string("seq")); err != nil { + return err + } + + if t.Seq >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Seq)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Seq-1)); err != nil { + return err + } + } + + // t.Time (string) (string) + if len("time") > 1000000 { + return xerrors.Errorf("Value in field \"time\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("time"))); err != nil { + return err + } + if _, err := cw.WriteString(string("time")); err != nil { + return err + } + + if len(t.Time) > 1000000 { + return xerrors.Errorf("Value in field t.Time was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Time))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Time)); err != nil { + return err + } + + // t.Active (bool) (bool) + if len("active") > 1000000 { + return xerrors.Errorf("Value in field \"active\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("active"))); err != nil { + return err + } + if _, err := cw.WriteString(string("active")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.Active); err != nil { + return err + } + + // t.Status (string) (string) + if t.Status != nil { + + if len("status") > 1000000 { + return xerrors.Errorf("Value in field \"status\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("status"))); err != nil { + return err + } + if _, err := cw.WriteString(string("status")); err != nil { + return err + } + + if t.Status == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Status) > 1000000 { + return xerrors.Errorf("Value in field t.Status was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Status))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Status)); err != nil { + return err + } + } + } + return nil +} + +func (t *SyncSubscribeRepos_Account) UnmarshalCBOR(r io.Reader) (err error) { + *t = SyncSubscribeRepos_Account{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SyncSubscribeRepos_Account: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Did (string) (string) + case "did": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Did = string(sval) + } + // t.Seq (int64) (int64) + case "seq": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Seq = int64(extraI) + } + // t.Time (string) (string) + case "time": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Time = string(sval) + } + // t.Active (bool) (bool) + case "active": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.Active = false + case 21: + t.Active = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.Status (string) (string) + case "status": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Status = (*string)(&sval) + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *SyncSubscribeRepos_Info) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 2 + + if t.Message == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Name (string) (string) + if len("name") > 1000000 { + return xerrors.Errorf("Value in field \"name\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := cw.WriteString(string("name")); err != nil { + return err + } + + if len(t.Name) > 1000000 { + return xerrors.Errorf("Value in field t.Name was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Name)); err != nil { + return err + } + + // t.Message (string) (string) + if t.Message != nil { + + if len("message") > 1000000 { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := cw.WriteString(string("message")); err != nil { + return err + } + + if t.Message == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Message) > 1000000 { + return xerrors.Errorf("Value in field t.Message was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Message))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Message)); err != nil { + return err + } + } + } + return nil +} + +func (t *SyncSubscribeRepos_Info) UnmarshalCBOR(r io.Reader) (err error) { + *t = SyncSubscribeRepos_Info{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SyncSubscribeRepos_Info: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 7) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Name (string) (string) + case "name": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Name = string(sval) + } + // t.Message (string) (string) + case "message": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Message = (*string)(&sval) + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *SyncSubscribeRepos_RepoOp) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 4 + + if t.Prev == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Cid (util.LexLink) (struct) + if len("cid") > 1000000 { + return xerrors.Errorf("Value in field \"cid\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil { + return err + } + if _, err := cw.WriteString(string("cid")); err != nil { + return err + } + + if err := t.Cid.MarshalCBOR(cw); err != nil { + return err + } + + // t.Path (string) (string) + if len("path") > 1000000 { + return xerrors.Errorf("Value in field \"path\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("path"))); err != nil { + return err + } + if _, err := cw.WriteString(string("path")); err != nil { + return err + } + + if len(t.Path) > 1000000 { + return xerrors.Errorf("Value in field t.Path was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Path))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Path)); err != nil { + return err + } + + // t.Prev (util.LexLink) (struct) + if t.Prev != nil { + + if len("prev") > 1000000 { + return xerrors.Errorf("Value in field \"prev\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("prev"))); err != nil { + return err + } + if _, err := cw.WriteString(string("prev")); err != nil { + return err + } + + if err := t.Prev.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Action (string) (string) + if len("action") > 1000000 { + return xerrors.Errorf("Value in field \"action\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("action"))); err != nil { + return err + } + if _, err := cw.WriteString(string("action")); err != nil { + return err + } + + if len(t.Action) > 1000000 { + return xerrors.Errorf("Value in field t.Action was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Action))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Action)); err != nil { + return err + } + return nil +} + +func (t *SyncSubscribeRepos_RepoOp) UnmarshalCBOR(r io.Reader) (err error) { + *t = SyncSubscribeRepos_RepoOp{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SyncSubscribeRepos_RepoOp: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Cid (util.LexLink) (struct) + case "cid": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Cid = new(util.LexLink) + if err := t.Cid.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Cid pointer: %w", err) + } + } + + } + // t.Path (string) (string) + case "path": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Path = string(sval) + } + // t.Prev (util.LexLink) (struct) + case "prev": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Prev = new(util.LexLink) + if err := t.Prev.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Prev pointer: %w", err) + } + } + + } + // t.Action (string) (string) + case "action": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Action = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *LabelDefs_SelfLabels) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("com.atproto.label.defs#selfLabels"))); err != nil { + return err + } + if _, err := cw.WriteString(string("com.atproto.label.defs#selfLabels")); err != nil { + return err + } + + // t.Values ([]*atproto.LabelDefs_SelfLabel) (slice) + if len("values") > 1000000 { + return xerrors.Errorf("Value in field \"values\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("values"))); err != nil { + return err + } + if _, err := cw.WriteString(string("values")); err != nil { + return err + } + + if len(t.Values) > 8192 { + return xerrors.Errorf("Slice value in field t.Values was too long") + } + + if t.Values == nil { + _, err := w.Write(cbg.CborNull) + if err != nil { + return err + } + } else { + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Values))); err != nil { + return err + } + for _, v := range t.Values { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + + } + return nil +} + +func (t *LabelDefs_SelfLabels) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelDefs_SelfLabels{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LabelDefs_SelfLabels: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Values ([]*atproto.LabelDefs_SelfLabel) (slice) + case "values": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Values: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + t.Values = make([]*LabelDefs_SelfLabel, extra) + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Values[i] = new(LabelDefs_SelfLabel) + if err := t.Values[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Values[i] pointer: %w", err) + } + } + + } + + } + } + + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *LabelDefs_SelfLabel) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Val (string) (string) + if len("val") > 1000000 { + return xerrors.Errorf("Value in field \"val\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("val"))); err != nil { + return err + } + if _, err := cw.WriteString(string("val")); err != nil { + return err + } + + if len(t.Val) > 1000000 { + return xerrors.Errorf("Value in field t.Val was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Val))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Val)); err != nil { + return err + } + return nil +} + +func (t *LabelDefs_SelfLabel) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelDefs_SelfLabel{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LabelDefs_SelfLabel: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 3) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Val (string) (string) + case "val": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Val = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *LabelDefs_Label) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 9 + + if t.Cid == nil { + fieldCount-- + } + + if t.Exp == nil { + fieldCount-- + } + + if t.Neg == nil { + fieldCount-- + } + + if t.Sig == nil { + fieldCount-- + } + + if t.Ver == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Cid (string) (string) + if t.Cid != nil { + + if len("cid") > 1000000 { + return xerrors.Errorf("Value in field \"cid\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil { + return err + } + if _, err := cw.WriteString(string("cid")); err != nil { + return err + } + + if t.Cid == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Cid) > 1000000 { + return xerrors.Errorf("Value in field t.Cid was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Cid))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Cid)); err != nil { + return err + } + } + } + + // t.Cts (string) (string) + if len("cts") > 1000000 { + return xerrors.Errorf("Value in field \"cts\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cts"))); err != nil { + return err + } + if _, err := cw.WriteString(string("cts")); err != nil { + return err + } + + if len(t.Cts) > 1000000 { + return xerrors.Errorf("Value in field t.Cts was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Cts))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Cts)); err != nil { + return err + } + + // t.Exp (string) (string) + if t.Exp != nil { + + if len("exp") > 1000000 { + return xerrors.Errorf("Value in field \"exp\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("exp"))); err != nil { + return err + } + if _, err := cw.WriteString(string("exp")); err != nil { + return err + } + + if t.Exp == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Exp) > 1000000 { + return xerrors.Errorf("Value in field t.Exp was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Exp))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Exp)); err != nil { + return err + } + } + } + + // t.Neg (bool) (bool) + if t.Neg != nil { + + if len("neg") > 1000000 { + return xerrors.Errorf("Value in field \"neg\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("neg"))); err != nil { + return err + } + if _, err := cw.WriteString(string("neg")); err != nil { + return err + } + + if t.Neg == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteBool(w, *t.Neg); err != nil { + return err + } + } + } + + // t.Sig (util.LexBytes) (slice) + if t.Sig != nil { + + if len("sig") > 1000000 { + return xerrors.Errorf("Value in field \"sig\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sig"))); err != nil { + return err + } + if _, err := cw.WriteString(string("sig")); err != nil { + return err + } + + if len(t.Sig) > 2097152 { + return xerrors.Errorf("Byte array in field t.Sig was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Sig))); err != nil { + return err + } + + if _, err := cw.Write(t.Sig); err != nil { + return err + } + + } + + // t.Src (string) (string) + if len("src") > 1000000 { + return xerrors.Errorf("Value in field \"src\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("src"))); err != nil { + return err + } + if _, err := cw.WriteString(string("src")); err != nil { + return err + } + + if len(t.Src) > 1000000 { + return xerrors.Errorf("Value in field t.Src was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Src))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Src)); err != nil { + return err + } + + // t.Uri (string) (string) + if len("uri") > 1000000 { + return xerrors.Errorf("Value in field \"uri\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { + return err + } + if _, err := cw.WriteString(string("uri")); err != nil { + return err + } + + if len(t.Uri) > 1000000 { + return xerrors.Errorf("Value in field t.Uri was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Uri)); err != nil { + return err + } + + // t.Val (string) (string) + if len("val") > 1000000 { + return xerrors.Errorf("Value in field \"val\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("val"))); err != nil { + return err + } + if _, err := cw.WriteString(string("val")); err != nil { + return err + } + + if len(t.Val) > 1000000 { + return xerrors.Errorf("Value in field t.Val was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Val))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Val)); err != nil { + return err + } + + // t.Ver (int64) (int64) + if t.Ver != nil { + + if len("ver") > 1000000 { + return xerrors.Errorf("Value in field \"ver\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ver"))); err != nil { + return err + } + if _, err := cw.WriteString(string("ver")); err != nil { + return err + } + + if t.Ver == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if *t.Ver >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(*t.Ver)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-*t.Ver-1)); err != nil { + return err + } + } + } + + } + return nil +} + +func (t *LabelDefs_Label) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelDefs_Label{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LabelDefs_Label: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 3) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Cid (string) (string) + case "cid": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Cid = (*string)(&sval) + } + } + // t.Cts (string) (string) + case "cts": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Cts = string(sval) + } + // t.Exp (string) (string) + case "exp": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Exp = (*string)(&sval) + } + } + // t.Neg (bool) (bool) + case "neg": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + + var val bool + switch extra { + case 20: + val = false + case 21: + val = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + t.Neg = &val + } + } + // t.Sig (util.LexBytes) (slice) + case "sig": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.Sig: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Sig = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Sig); err != nil { + return err + } + + // t.Src (string) (string) + case "src": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Src = string(sval) + } + // t.Uri (string) (string) + case "uri": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Uri = string(sval) + } + // t.Val (string) (string) + case "val": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Val = string(sval) + } + // t.Ver (int64) (int64) + case "ver": + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Ver = (*int64)(&extraI) + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *LabelSubscribeLabels_Labels) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Seq (int64) (int64) + if len("seq") > 1000000 { + return xerrors.Errorf("Value in field \"seq\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("seq"))); err != nil { + return err + } + if _, err := cw.WriteString(string("seq")); err != nil { + return err + } + + if t.Seq >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Seq)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Seq-1)); err != nil { + return err + } + } + + // t.Labels ([]*atproto.LabelDefs_Label) (slice) + if len("labels") > 1000000 { + return xerrors.Errorf("Value in field \"labels\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("labels"))); err != nil { + return err + } + if _, err := cw.WriteString(string("labels")); err != nil { + return err + } + + if len(t.Labels) > 8192 { + return xerrors.Errorf("Slice value in field t.Labels was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Labels))); err != nil { + return err + } + for _, v := range t.Labels { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + return nil +} + +func (t *LabelSubscribeLabels_Labels) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelSubscribeLabels_Labels{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LabelSubscribeLabels_Labels: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Seq (int64) (int64) + case "seq": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Seq = int64(extraI) + } + // t.Labels ([]*atproto.LabelDefs_Label) (slice) + case "labels": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Labels: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Labels = make([]*LabelDefs_Label, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Labels[i] = new(LabelDefs_Label) + if err := t.Labels[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Labels[i] pointer: %w", err) + } + } + + } + + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *LabelSubscribeLabels_Info) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 2 + + if t.Message == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Name (string) (string) + if len("name") > 1000000 { + return xerrors.Errorf("Value in field \"name\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := cw.WriteString(string("name")); err != nil { + return err + } + + if len(t.Name) > 1000000 { + return xerrors.Errorf("Value in field t.Name was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Name)); err != nil { + return err + } + + // t.Message (string) (string) + if t.Message != nil { + + if len("message") > 1000000 { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := cw.WriteString(string("message")); err != nil { + return err + } + + if t.Message == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Message) > 1000000 { + return xerrors.Errorf("Value in field t.Message was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Message))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Message)); err != nil { + return err + } + } + } + return nil +} + +func (t *LabelSubscribeLabels_Info) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelSubscribeLabels_Info{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LabelSubscribeLabels_Info: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 7) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Name (string) (string) + case "name": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Name = string(sval) + } + // t.Message (string) (string) + case "message": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Message = (*string)(&sval) + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *LabelDefs_LabelValueDefinition) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 6 + + if t.AdultOnly == nil { + fieldCount-- + } + + if t.DefaultSetting == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Blurs (string) (string) + if len("blurs") > 1000000 { + return xerrors.Errorf("Value in field \"blurs\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blurs"))); err != nil { + return err + } + if _, err := cw.WriteString(string("blurs")); err != nil { + return err + } + + if len(t.Blurs) > 1000000 { + return xerrors.Errorf("Value in field t.Blurs was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Blurs))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Blurs)); err != nil { + return err + } + + // t.Locales ([]*atproto.LabelDefs_LabelValueDefinitionStrings) (slice) + if len("locales") > 1000000 { + return xerrors.Errorf("Value in field \"locales\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("locales"))); err != nil { + return err + } + if _, err := cw.WriteString(string("locales")); err != nil { + return err + } + + if len(t.Locales) > 8192 { + return xerrors.Errorf("Slice value in field t.Locales was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Locales))); err != nil { + return err + } + for _, v := range t.Locales { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + + // t.Severity (string) (string) + if len("severity") > 1000000 { + return xerrors.Errorf("Value in field \"severity\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("severity"))); err != nil { + return err + } + if _, err := cw.WriteString(string("severity")); err != nil { + return err + } + + if len(t.Severity) > 1000000 { + return xerrors.Errorf("Value in field t.Severity was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Severity))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Severity)); err != nil { + return err + } + + // t.AdultOnly (bool) (bool) + if t.AdultOnly != nil { + + if len("adultOnly") > 1000000 { + return xerrors.Errorf("Value in field \"adultOnly\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("adultOnly"))); err != nil { + return err + } + if _, err := cw.WriteString(string("adultOnly")); err != nil { + return err + } + + if t.AdultOnly == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteBool(w, *t.AdultOnly); err != nil { + return err + } + } + } + + // t.Identifier (string) (string) + if len("identifier") > 1000000 { + return xerrors.Errorf("Value in field \"identifier\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("identifier"))); err != nil { + return err + } + if _, err := cw.WriteString(string("identifier")); err != nil { + return err + } + + if len(t.Identifier) > 1000000 { + return xerrors.Errorf("Value in field t.Identifier was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Identifier))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Identifier)); err != nil { + return err + } + + // t.DefaultSetting (string) (string) + if t.DefaultSetting != nil { + + if len("defaultSetting") > 1000000 { + return xerrors.Errorf("Value in field \"defaultSetting\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("defaultSetting"))); err != nil { + return err + } + if _, err := cw.WriteString(string("defaultSetting")); err != nil { + return err + } + + if t.DefaultSetting == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.DefaultSetting) > 1000000 { + return xerrors.Errorf("Value in field t.DefaultSetting was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.DefaultSetting))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.DefaultSetting)); err != nil { + return err + } + } + } + return nil +} + +func (t *LabelDefs_LabelValueDefinition) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelDefs_LabelValueDefinition{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LabelDefs_LabelValueDefinition: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 14) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Blurs (string) (string) + case "blurs": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Blurs = string(sval) + } + // t.Locales ([]*atproto.LabelDefs_LabelValueDefinitionStrings) (slice) + case "locales": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Locales: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Locales = make([]*LabelDefs_LabelValueDefinitionStrings, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Locales[i] = new(LabelDefs_LabelValueDefinitionStrings) + if err := t.Locales[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Locales[i] pointer: %w", err) + } + } + + } + + } + } + // t.Severity (string) (string) + case "severity": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Severity = string(sval) + } + // t.AdultOnly (bool) (bool) + case "adultOnly": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + + var val bool + switch extra { + case 20: + val = false + case 21: + val = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + t.AdultOnly = &val + } + } + // t.Identifier (string) (string) + case "identifier": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Identifier = string(sval) + } + // t.DefaultSetting (string) (string) + case "defaultSetting": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.DefaultSetting = (*string)(&sval) + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *LabelDefs_LabelValueDefinitionStrings) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Lang (string) (string) + if len("lang") > 1000000 { + return xerrors.Errorf("Value in field \"lang\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("lang"))); err != nil { + return err + } + if _, err := cw.WriteString(string("lang")); err != nil { + return err + } + + if len(t.Lang) > 1000000 { + return xerrors.Errorf("Value in field t.Lang was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Lang))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Lang)); err != nil { + return err + } + + // t.Name (string) (string) + if len("name") > 1000000 { + return xerrors.Errorf("Value in field \"name\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := cw.WriteString(string("name")); err != nil { + return err + } + + if len(t.Name) > 1000000 { + return xerrors.Errorf("Value in field t.Name was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Name)); err != nil { + return err + } + + // t.Description (string) (string) + if len("description") > 1000000 { + return xerrors.Errorf("Value in field \"description\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("description"))); err != nil { + return err + } + if _, err := cw.WriteString(string("description")); err != nil { + return err + } + + if len(t.Description) > 1000000 { + return xerrors.Errorf("Value in field t.Description was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Description))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Description)); err != nil { + return err + } + return nil +} + +func (t *LabelDefs_LabelValueDefinitionStrings) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelDefs_LabelValueDefinitionStrings{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LabelDefs_LabelValueDefinitionStrings: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 11) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Lang (string) (string) + case "lang": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Lang = string(sval) + } + // t.Name (string) (string) + case "name": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Name = string(sval) + } + // t.Description (string) (string) + case "description": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Description = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/api/atproto/handleresolve.go b/api/atproto/handleresolve.go deleted file mode 100644 index bfa82d493..000000000 --- a/api/atproto/handleresolve.go +++ /dev/null @@ -1,30 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.handle.resolve - -func init() { -} - -type HandleResolve_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Did string `json:"did" cborgen:"did"` -} - -func HandleResolve(ctx context.Context, c *xrpc.Client, handle string) (*HandleResolve_Output, error) { - var out HandleResolve_Output - - params := map[string]interface{}{ - "handle": handle, - } - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.handle.resolve", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/identitydefs.go b/api/atproto/identitydefs.go new file mode 100644 index 000000000..e5cf8bf58 --- /dev/null +++ b/api/atproto/identitydefs.go @@ -0,0 +1,14 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.defs + +package atproto + +// IdentityDefs_IdentityInfo is a "identityInfo" in the com.atproto.identity.defs schema. +type IdentityDefs_IdentityInfo struct { + Did string `json:"did" cborgen:"did"` + // didDoc: The complete DID document for the identity. + DidDoc interface{} `json:"didDoc" cborgen:"didDoc"` + // handle: The validated handle of the account; or 'handle.invalid' if the handle did not bi-directionally match the DID document. + Handle string `json:"handle" cborgen:"handle"` +} diff --git a/api/atproto/identitygetRecommendedDidCredentials.go b/api/atproto/identitygetRecommendedDidCredentials.go new file mode 100644 index 000000000..3a524b1b4 --- /dev/null +++ b/api/atproto/identitygetRecommendedDidCredentials.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.getRecommendedDidCredentials + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentityGetRecommendedDidCredentials_Output is the output of a com.atproto.identity.getRecommendedDidCredentials call. +type IdentityGetRecommendedDidCredentials_Output struct { + AlsoKnownAs []string `json:"alsoKnownAs,omitempty" cborgen:"alsoKnownAs,omitempty"` + // rotationKeys: Recommended rotation keys for PLC dids. Should be undefined (or ignored) for did:webs. + RotationKeys []string `json:"rotationKeys,omitempty" cborgen:"rotationKeys,omitempty"` + Services *lexutil.LexiconTypeDecoder `json:"services,omitempty" cborgen:"services,omitempty"` + VerificationMethods *lexutil.LexiconTypeDecoder `json:"verificationMethods,omitempty" cborgen:"verificationMethods,omitempty"` +} + +// IdentityGetRecommendedDidCredentials calls the XRPC method "com.atproto.identity.getRecommendedDidCredentials". +func IdentityGetRecommendedDidCredentials(ctx context.Context, c lexutil.LexClient) (*IdentityGetRecommendedDidCredentials_Output, error) { + var out IdentityGetRecommendedDidCredentials_Output + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.identity.getRecommendedDidCredentials", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/identityrefreshIdentity.go b/api/atproto/identityrefreshIdentity.go new file mode 100644 index 000000000..321d89623 --- /dev/null +++ b/api/atproto/identityrefreshIdentity.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.refreshIdentity + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentityRefreshIdentity_Input is the input argument to a com.atproto.identity.refreshIdentity call. +type IdentityRefreshIdentity_Input struct { + Identifier string `json:"identifier" cborgen:"identifier"` +} + +// IdentityRefreshIdentity calls the XRPC method "com.atproto.identity.refreshIdentity". +func IdentityRefreshIdentity(ctx context.Context, c lexutil.LexClient, input *IdentityRefreshIdentity_Input) (*IdentityDefs_IdentityInfo, error) { + var out IdentityDefs_IdentityInfo + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.identity.refreshIdentity", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/identityrequestPlcOperationSignature.go b/api/atproto/identityrequestPlcOperationSignature.go new file mode 100644 index 000000000..d54857310 --- /dev/null +++ b/api/atproto/identityrequestPlcOperationSignature.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.requestPlcOperationSignature + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentityRequestPlcOperationSignature calls the XRPC method "com.atproto.identity.requestPlcOperationSignature". +func IdentityRequestPlcOperationSignature(ctx context.Context, c lexutil.LexClient) error { + if err := c.LexDo(ctx, lexutil.Procedure, "", "com.atproto.identity.requestPlcOperationSignature", nil, nil, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/identityresolveDid.go b/api/atproto/identityresolveDid.go new file mode 100644 index 000000000..b55abb9c1 --- /dev/null +++ b/api/atproto/identityresolveDid.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.resolveDid + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentityResolveDid_Output is the output of a com.atproto.identity.resolveDid call. +type IdentityResolveDid_Output struct { + // didDoc: The complete DID document for the identity. + DidDoc interface{} `json:"didDoc" cborgen:"didDoc"` +} + +// IdentityResolveDid calls the XRPC method "com.atproto.identity.resolveDid". +// +// did: DID to resolve. +func IdentityResolveDid(ctx context.Context, c lexutil.LexClient, did string) (*IdentityResolveDid_Output, error) { + var out IdentityResolveDid_Output + + params := map[string]interface{}{} + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.identity.resolveDid", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/identityresolveHandle.go b/api/atproto/identityresolveHandle.go new file mode 100644 index 000000000..3e825ebe2 --- /dev/null +++ b/api/atproto/identityresolveHandle.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.resolveHandle + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentityResolveHandle_Output is the output of a com.atproto.identity.resolveHandle call. +type IdentityResolveHandle_Output struct { + Did string `json:"did" cborgen:"did"` +} + +// IdentityResolveHandle calls the XRPC method "com.atproto.identity.resolveHandle". +// +// handle: The handle to resolve. +func IdentityResolveHandle(ctx context.Context, c lexutil.LexClient, handle string) (*IdentityResolveHandle_Output, error) { + var out IdentityResolveHandle_Output + + params := map[string]interface{}{} + params["handle"] = handle + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.identity.resolveHandle", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/identityresolveIdentity.go b/api/atproto/identityresolveIdentity.go new file mode 100644 index 000000000..9699d2322 --- /dev/null +++ b/api/atproto/identityresolveIdentity.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.resolveIdentity + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentityResolveIdentity calls the XRPC method "com.atproto.identity.resolveIdentity". +// +// identifier: Handle or DID to resolve. +func IdentityResolveIdentity(ctx context.Context, c lexutil.LexClient, identifier string) (*IdentityDefs_IdentityInfo, error) { + var out IdentityDefs_IdentityInfo + + params := map[string]interface{}{} + params["identifier"] = identifier + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.identity.resolveIdentity", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/identitysignPlcOperation.go b/api/atproto/identitysignPlcOperation.go new file mode 100644 index 000000000..8c1702a0d --- /dev/null +++ b/api/atproto/identitysignPlcOperation.go @@ -0,0 +1,37 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.signPlcOperation + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentitySignPlcOperation_Input is the input argument to a com.atproto.identity.signPlcOperation call. +type IdentitySignPlcOperation_Input struct { + AlsoKnownAs []string `json:"alsoKnownAs,omitempty" cborgen:"alsoKnownAs,omitempty"` + RotationKeys []string `json:"rotationKeys,omitempty" cborgen:"rotationKeys,omitempty"` + Services *lexutil.LexiconTypeDecoder `json:"services,omitempty" cborgen:"services,omitempty"` + // token: A token received through com.atproto.identity.requestPlcOperationSignature + Token *string `json:"token,omitempty" cborgen:"token,omitempty"` + VerificationMethods *lexutil.LexiconTypeDecoder `json:"verificationMethods,omitempty" cborgen:"verificationMethods,omitempty"` +} + +// IdentitySignPlcOperation_Output is the output of a com.atproto.identity.signPlcOperation call. +type IdentitySignPlcOperation_Output struct { + // operation: A signed DID PLC operation. + Operation *lexutil.LexiconTypeDecoder `json:"operation" cborgen:"operation"` +} + +// IdentitySignPlcOperation calls the XRPC method "com.atproto.identity.signPlcOperation". +func IdentitySignPlcOperation(ctx context.Context, c lexutil.LexClient, input *IdentitySignPlcOperation_Input) (*IdentitySignPlcOperation_Output, error) { + var out IdentitySignPlcOperation_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.identity.signPlcOperation", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/identitysubmitPlcOperation.go b/api/atproto/identitysubmitPlcOperation.go new file mode 100644 index 000000000..e5e947164 --- /dev/null +++ b/api/atproto/identitysubmitPlcOperation.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.submitPlcOperation + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentitySubmitPlcOperation_Input is the input argument to a com.atproto.identity.submitPlcOperation call. +type IdentitySubmitPlcOperation_Input struct { + Operation *lexutil.LexiconTypeDecoder `json:"operation" cborgen:"operation"` +} + +// IdentitySubmitPlcOperation calls the XRPC method "com.atproto.identity.submitPlcOperation". +func IdentitySubmitPlcOperation(ctx context.Context, c lexutil.LexClient, input *IdentitySubmitPlcOperation_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.identity.submitPlcOperation", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/identityupdateHandle.go b/api/atproto/identityupdateHandle.go new file mode 100644 index 000000000..248a1327a --- /dev/null +++ b/api/atproto/identityupdateHandle.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.identity.updateHandle + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// IdentityUpdateHandle_Input is the input argument to a com.atproto.identity.updateHandle call. +type IdentityUpdateHandle_Input struct { + // handle: The new handle. + Handle string `json:"handle" cborgen:"handle"` +} + +// IdentityUpdateHandle calls the XRPC method "com.atproto.identity.updateHandle". +func IdentityUpdateHandle(ctx context.Context, c lexutil.LexClient, input *IdentityUpdateHandle_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.identity.updateHandle", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/labeldefs.go b/api/atproto/labeldefs.go new file mode 100644 index 000000000..69008b86f --- /dev/null +++ b/api/atproto/labeldefs.go @@ -0,0 +1,78 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.label.defs + +package atproto + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// LabelDefs_Label is a "label" in the com.atproto.label.defs schema. +// +// Metadata tag on an atproto resource (eg, repo or record). +type LabelDefs_Label struct { + // cid: Optionally, CID specifying the specific version of 'uri' resource this label applies to. + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + // cts: Timestamp when this label was created. + Cts string `json:"cts" cborgen:"cts"` + // exp: Timestamp at which this label expires (no longer applies). + Exp *string `json:"exp,omitempty" cborgen:"exp,omitempty"` + // neg: If true, this is a negation label, overwriting a previous label. + Neg *bool `json:"neg,omitempty" cborgen:"neg,omitempty"` + // sig: Signature of dag-cbor encoded label. + Sig lexutil.LexBytes `json:"sig,omitempty" cborgen:"sig,omitempty"` + // src: DID of the actor who created this label. + Src string `json:"src" cborgen:"src"` + // uri: AT URI of the record, repository (account), or other resource that this label applies to. + Uri string `json:"uri" cborgen:"uri"` + // val: The short string name of the value or type of this label. + Val string `json:"val" cborgen:"val"` + // ver: The AT Protocol version of the label object. + Ver *int64 `json:"ver,omitempty" cborgen:"ver,omitempty"` +} + +// LabelDefs_LabelValueDefinition is a "labelValueDefinition" in the com.atproto.label.defs schema. +// +// Declares a label value and its expected interpretations and behaviors. +type LabelDefs_LabelValueDefinition struct { + // adultOnly: Does the user need to have adult content enabled in order to configure this label? + AdultOnly *bool `json:"adultOnly,omitempty" cborgen:"adultOnly,omitempty"` + // blurs: What should this label hide in the UI, if applied? 'content' hides all of the target; 'media' hides the images/video/audio; 'none' hides nothing. + Blurs string `json:"blurs" cborgen:"blurs"` + // defaultSetting: The default setting for this label. + DefaultSetting *string `json:"defaultSetting,omitempty" cborgen:"defaultSetting,omitempty"` + // identifier: The value of the label being defined. Must only include lowercase ascii and the '-' character ([a-z-]+). + Identifier string `json:"identifier" cborgen:"identifier"` + Locales []*LabelDefs_LabelValueDefinitionStrings `json:"locales" cborgen:"locales"` + // severity: How should a client visually convey this label? 'inform' means neutral and informational; 'alert' means negative and warning; 'none' means show nothing. + Severity string `json:"severity" cborgen:"severity"` +} + +// LabelDefs_LabelValueDefinitionStrings is a "labelValueDefinitionStrings" in the com.atproto.label.defs schema. +// +// Strings which describe the label in the UI, localized into a specific language. +type LabelDefs_LabelValueDefinitionStrings struct { + // description: A longer description of what the label means and why it might be applied. + Description string `json:"description" cborgen:"description"` + // lang: The code of the language these strings are written in. + Lang string `json:"lang" cborgen:"lang"` + // name: A short human-readable name for the label. + Name string `json:"name" cborgen:"name"` +} + +// LabelDefs_SelfLabel is a "selfLabel" in the com.atproto.label.defs schema. +// +// Metadata tag on an atproto record, published by the author within the record. Note that schemas should use #selfLabels, not #selfLabel. +type LabelDefs_SelfLabel struct { + // val: The short string name of the value or type of this label. + Val string `json:"val" cborgen:"val"` +} + +// LabelDefs_SelfLabels is a "selfLabels" in the com.atproto.label.defs schema. +// +// Metadata tags on an atproto record, published by the author within the record. +type LabelDefs_SelfLabels struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.label.defs#selfLabels"` + Values []*LabelDefs_SelfLabel `json:"values" cborgen:"values,preservenil"` +} diff --git a/api/atproto/labelqueryLabels.go b/api/atproto/labelqueryLabels.go new file mode 100644 index 000000000..085139fa0 --- /dev/null +++ b/api/atproto/labelqueryLabels.go @@ -0,0 +1,42 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.label.queryLabels + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// LabelQueryLabels_Output is the output of a com.atproto.label.queryLabels call. +type LabelQueryLabels_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Labels []*LabelDefs_Label `json:"labels" cborgen:"labels"` +} + +// LabelQueryLabels calls the XRPC method "com.atproto.label.queryLabels". +// +// sources: Optional list of label sources (DIDs) to filter on. +// uriPatterns: List of AT URI patterns to match (boolean 'OR'). Each may be a prefix (ending with '*'; will match inclusive of the string leading to '*'), or a full URI. +func LabelQueryLabels(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, sources []string, uriPatterns []string) (*LabelQueryLabels_Output, error) { + var out LabelQueryLabels_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if len(sources) != 0 { + params["sources"] = sources + } + params["uriPatterns"] = uriPatterns + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.label.queryLabels", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/labelsubscribeLabels.go b/api/atproto/labelsubscribeLabels.go new file mode 100644 index 000000000..0090c82b8 --- /dev/null +++ b/api/atproto/labelsubscribeLabels.go @@ -0,0 +1,17 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.label.subscribeLabels + +package atproto + +// LabelSubscribeLabels_Info is a "info" in the com.atproto.label.subscribeLabels schema. +type LabelSubscribeLabels_Info struct { + Message *string `json:"message,omitempty" cborgen:"message,omitempty"` + Name string `json:"name" cborgen:"name"` +} + +// LabelSubscribeLabels_Labels is a "labels" in the com.atproto.label.subscribeLabels schema. +type LabelSubscribeLabels_Labels struct { + Labels []*LabelDefs_Label `json:"labels" cborgen:"labels"` + Seq int64 `json:"seq" cborgen:"seq"` +} diff --git a/api/atproto/lexiconschema.go b/api/atproto/lexiconschema.go new file mode 100644 index 000000000..d10cba044 --- /dev/null +++ b/api/atproto/lexiconschema.go @@ -0,0 +1,19 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.lexicon.schema + +package atproto + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("com.atproto.lexicon.schema", &LexiconSchema{}) +} + +type LexiconSchema struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.lexicon.schema"` + // lexicon: Indicates the 'version' of the Lexicon language. Must be '1' for the current atproto/Lexicon schema system. + Lexicon int64 `json:"lexicon" cborgen:"lexicon"` +} diff --git a/api/atproto/moderationcreateReport.go b/api/atproto/moderationcreateReport.go new file mode 100644 index 000000000..d017baeb9 --- /dev/null +++ b/api/atproto/moderationcreateReport.go @@ -0,0 +1,123 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.moderation.createReport + +package atproto + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationCreateReport_Input is the input argument to a com.atproto.moderation.createReport call. +type ModerationCreateReport_Input struct { + ModTool *ModerationCreateReport_ModTool `json:"modTool,omitempty" cborgen:"modTool,omitempty"` + // reason: Additional context about the content and violation. + Reason *string `json:"reason,omitempty" cborgen:"reason,omitempty"` + // reasonType: Indicates the broad category of violation the report is for. + ReasonType *string `json:"reasonType" cborgen:"reasonType"` + Subject *ModerationCreateReport_Input_Subject `json:"subject" cborgen:"subject"` +} + +type ModerationCreateReport_Input_Subject struct { + AdminDefs_RepoRef *AdminDefs_RepoRef + RepoStrongRef *RepoStrongRef +} + +func (t *ModerationCreateReport_Input_Subject) MarshalJSON() ([]byte, error) { + if t.AdminDefs_RepoRef != nil { + t.AdminDefs_RepoRef.LexiconTypeID = "com.atproto.admin.defs#repoRef" + return json.Marshal(t.AdminDefs_RepoRef) + } + if t.RepoStrongRef != nil { + t.RepoStrongRef.LexiconTypeID = "com.atproto.repo.strongRef" + return json.Marshal(t.RepoStrongRef) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationCreateReport_Input_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.admin.defs#repoRef": + t.AdminDefs_RepoRef = new(AdminDefs_RepoRef) + return json.Unmarshal(b, t.AdminDefs_RepoRef) + case "com.atproto.repo.strongRef": + t.RepoStrongRef = new(RepoStrongRef) + return json.Unmarshal(b, t.RepoStrongRef) + default: + return nil + } +} + +// ModerationCreateReport_ModTool is a "modTool" in the com.atproto.moderation.createReport schema. +// +// Moderation tool information for tracing the source of the action +type ModerationCreateReport_ModTool struct { + // meta: Additional arbitrary metadata about the source + Meta *interface{} `json:"meta,omitempty" cborgen:"meta,omitempty"` + // name: Name/identifier of the source (e.g., 'bsky-app/android', 'bsky-web/chrome') + Name string `json:"name" cborgen:"name"` +} + +// ModerationCreateReport_Output is the output of a com.atproto.moderation.createReport call. +type ModerationCreateReport_Output struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Id int64 `json:"id" cborgen:"id"` + Reason *string `json:"reason,omitempty" cborgen:"reason,omitempty"` + ReasonType *string `json:"reasonType" cborgen:"reasonType"` + ReportedBy string `json:"reportedBy" cborgen:"reportedBy"` + Subject *ModerationCreateReport_Output_Subject `json:"subject" cborgen:"subject"` +} + +type ModerationCreateReport_Output_Subject struct { + AdminDefs_RepoRef *AdminDefs_RepoRef + RepoStrongRef *RepoStrongRef +} + +func (t *ModerationCreateReport_Output_Subject) MarshalJSON() ([]byte, error) { + if t.AdminDefs_RepoRef != nil { + t.AdminDefs_RepoRef.LexiconTypeID = "com.atproto.admin.defs#repoRef" + return json.Marshal(t.AdminDefs_RepoRef) + } + if t.RepoStrongRef != nil { + t.RepoStrongRef.LexiconTypeID = "com.atproto.repo.strongRef" + return json.Marshal(t.RepoStrongRef) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationCreateReport_Output_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.admin.defs#repoRef": + t.AdminDefs_RepoRef = new(AdminDefs_RepoRef) + return json.Unmarshal(b, t.AdminDefs_RepoRef) + case "com.atproto.repo.strongRef": + t.RepoStrongRef = new(RepoStrongRef) + return json.Unmarshal(b, t.RepoStrongRef) + default: + return nil + } +} + +// ModerationCreateReport calls the XRPC method "com.atproto.moderation.createReport". +func ModerationCreateReport(ctx context.Context, c lexutil.LexClient, input *ModerationCreateReport_Input) (*ModerationCreateReport_Output, error) { + var out ModerationCreateReport_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.moderation.createReport", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/moderationdefs.go b/api/atproto/moderationdefs.go new file mode 100644 index 000000000..827679345 --- /dev/null +++ b/api/atproto/moderationdefs.go @@ -0,0 +1,5 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.moderation.defs + +package atproto diff --git a/api/atproto/peeringfollow.go b/api/atproto/peeringfollow.go deleted file mode 100644 index 620b562ac..000000000 --- a/api/atproto/peeringfollow.go +++ /dev/null @@ -1,25 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.peering.follow - -func init() { -} - -type PeeringFollow_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Users []string `json:"users" cborgen:"users"` -} - -func PeeringFollow(ctx context.Context, c *xrpc.Client, input *PeeringFollow_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.peering.follow", nil, input, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/peeringinit.go b/api/atproto/peeringinit.go deleted file mode 100644 index 9d284366d..000000000 --- a/api/atproto/peeringinit.go +++ /dev/null @@ -1,25 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.peering.init - -func init() { -} - -type PeeringInit_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Pds string `json:"pds" cborgen:"pds"` -} - -func PeeringInit(ctx context.Context, c *xrpc.Client, input *PeeringInit_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.peering.init", nil, input, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/peeringlist.go b/api/atproto/peeringlist.go deleted file mode 100644 index 410f7adee..000000000 --- a/api/atproto/peeringlist.go +++ /dev/null @@ -1,32 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.peering.list - -func init() { -} - -type PeeringList_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Peerings []*PeeringList_Peering `json:"peerings" cborgen:"peerings"` -} - -type PeeringList_Peering struct { - LexiconTypeID string `json:"$type,omitempty"` - Host *string `json:"host" cborgen:"host"` - Status *string `json:"status" cborgen:"status"` -} - -func PeeringList(ctx context.Context, c *xrpc.Client) (*PeeringList_Output, error) { - var out PeeringList_Output - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.peering.list", nil, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/peeringpropose.go b/api/atproto/peeringpropose.go deleted file mode 100644 index 656b74706..000000000 --- a/api/atproto/peeringpropose.go +++ /dev/null @@ -1,37 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.peering.propose - -func init() { -} - -type PeeringPropose_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Proposal *PeeringPropose_Proposal `json:"proposal" cborgen:"proposal"` - Signature string `json:"signature" cborgen:"signature"` -} - -type PeeringPropose_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Rejected bool `json:"rejected" cborgen:"rejected"` -} - -type PeeringPropose_Proposal struct { - LexiconTypeID string `json:"$type,omitempty"` - Proposer *string `json:"proposer" cborgen:"proposer"` -} - -func PeeringPropose(ctx context.Context, c *xrpc.Client, input *PeeringPropose_Input) (*PeeringPropose_Output, error) { - var out PeeringPropose_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.peering.propose", nil, input, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/repoapplyWrites.go b/api/atproto/repoapplyWrites.go new file mode 100644 index 000000000..599d21425 --- /dev/null +++ b/api/atproto/repoapplyWrites.go @@ -0,0 +1,177 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.applyWrites + +package atproto + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// RepoApplyWrites_Create is a "create" in the com.atproto.repo.applyWrites schema. +// +// Operation which creates a new record. +type RepoApplyWrites_Create struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.repo.applyWrites#create"` + Collection string `json:"collection" cborgen:"collection"` + // rkey: NOTE: maxLength is redundant with record-key format. Keeping it temporarily to ensure backwards compatibility. + Rkey *string `json:"rkey,omitempty" cborgen:"rkey,omitempty"` + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` +} + +// RepoApplyWrites_CreateResult is a "createResult" in the com.atproto.repo.applyWrites schema. +type RepoApplyWrites_CreateResult struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.repo.applyWrites#createResult"` + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + +// RepoApplyWrites_Delete is a "delete" in the com.atproto.repo.applyWrites schema. +// +// Operation which deletes an existing record. +type RepoApplyWrites_Delete struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.repo.applyWrites#delete"` + Collection string `json:"collection" cborgen:"collection"` + Rkey string `json:"rkey" cborgen:"rkey"` +} + +// RepoApplyWrites_DeleteResult is a "deleteResult" in the com.atproto.repo.applyWrites schema. +type RepoApplyWrites_DeleteResult struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.repo.applyWrites#deleteResult"` +} + +// RepoApplyWrites_Input is the input argument to a com.atproto.repo.applyWrites call. +type RepoApplyWrites_Input struct { + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // swapCommit: If provided, the entire operation will fail if the current repo commit CID does not match this value. Used to prevent conflicting repo mutations. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // validate: Can be set to 'false' to skip Lexicon schema validation of record data across all operations, 'true' to require it, or leave unset to validate only for known Lexicons. + Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` + Writes []*RepoApplyWrites_Input_Writes_Elem `json:"writes" cborgen:"writes"` +} + +type RepoApplyWrites_Input_Writes_Elem struct { + RepoApplyWrites_Create *RepoApplyWrites_Create + RepoApplyWrites_Update *RepoApplyWrites_Update + RepoApplyWrites_Delete *RepoApplyWrites_Delete +} + +func (t *RepoApplyWrites_Input_Writes_Elem) MarshalJSON() ([]byte, error) { + if t.RepoApplyWrites_Create != nil { + t.RepoApplyWrites_Create.LexiconTypeID = "com.atproto.repo.applyWrites#create" + return json.Marshal(t.RepoApplyWrites_Create) + } + if t.RepoApplyWrites_Update != nil { + t.RepoApplyWrites_Update.LexiconTypeID = "com.atproto.repo.applyWrites#update" + return json.Marshal(t.RepoApplyWrites_Update) + } + if t.RepoApplyWrites_Delete != nil { + t.RepoApplyWrites_Delete.LexiconTypeID = "com.atproto.repo.applyWrites#delete" + return json.Marshal(t.RepoApplyWrites_Delete) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *RepoApplyWrites_Input_Writes_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.repo.applyWrites#create": + t.RepoApplyWrites_Create = new(RepoApplyWrites_Create) + return json.Unmarshal(b, t.RepoApplyWrites_Create) + case "com.atproto.repo.applyWrites#update": + t.RepoApplyWrites_Update = new(RepoApplyWrites_Update) + return json.Unmarshal(b, t.RepoApplyWrites_Update) + case "com.atproto.repo.applyWrites#delete": + t.RepoApplyWrites_Delete = new(RepoApplyWrites_Delete) + return json.Unmarshal(b, t.RepoApplyWrites_Delete) + default: + return fmt.Errorf("closed unions must match a listed schema") + } +} + +// RepoApplyWrites_Output is the output of a com.atproto.repo.applyWrites call. +type RepoApplyWrites_Output struct { + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Results []*RepoApplyWrites_Output_Results_Elem `json:"results,omitempty" cborgen:"results,omitempty"` +} + +type RepoApplyWrites_Output_Results_Elem struct { + RepoApplyWrites_CreateResult *RepoApplyWrites_CreateResult + RepoApplyWrites_UpdateResult *RepoApplyWrites_UpdateResult + RepoApplyWrites_DeleteResult *RepoApplyWrites_DeleteResult +} + +func (t *RepoApplyWrites_Output_Results_Elem) MarshalJSON() ([]byte, error) { + if t.RepoApplyWrites_CreateResult != nil { + t.RepoApplyWrites_CreateResult.LexiconTypeID = "com.atproto.repo.applyWrites#createResult" + return json.Marshal(t.RepoApplyWrites_CreateResult) + } + if t.RepoApplyWrites_UpdateResult != nil { + t.RepoApplyWrites_UpdateResult.LexiconTypeID = "com.atproto.repo.applyWrites#updateResult" + return json.Marshal(t.RepoApplyWrites_UpdateResult) + } + if t.RepoApplyWrites_DeleteResult != nil { + t.RepoApplyWrites_DeleteResult.LexiconTypeID = "com.atproto.repo.applyWrites#deleteResult" + return json.Marshal(t.RepoApplyWrites_DeleteResult) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *RepoApplyWrites_Output_Results_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.repo.applyWrites#createResult": + t.RepoApplyWrites_CreateResult = new(RepoApplyWrites_CreateResult) + return json.Unmarshal(b, t.RepoApplyWrites_CreateResult) + case "com.atproto.repo.applyWrites#updateResult": + t.RepoApplyWrites_UpdateResult = new(RepoApplyWrites_UpdateResult) + return json.Unmarshal(b, t.RepoApplyWrites_UpdateResult) + case "com.atproto.repo.applyWrites#deleteResult": + t.RepoApplyWrites_DeleteResult = new(RepoApplyWrites_DeleteResult) + return json.Unmarshal(b, t.RepoApplyWrites_DeleteResult) + default: + return fmt.Errorf("closed unions must match a listed schema") + } +} + +// RepoApplyWrites_Update is a "update" in the com.atproto.repo.applyWrites schema. +// +// Operation which updates an existing record. +type RepoApplyWrites_Update struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.repo.applyWrites#update"` + Collection string `json:"collection" cborgen:"collection"` + Rkey string `json:"rkey" cborgen:"rkey"` + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` +} + +// RepoApplyWrites_UpdateResult is a "updateResult" in the com.atproto.repo.applyWrites schema. +type RepoApplyWrites_UpdateResult struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.repo.applyWrites#updateResult"` + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` +} + +// RepoApplyWrites calls the XRPC method "com.atproto.repo.applyWrites". +func RepoApplyWrites(ctx context.Context, c lexutil.LexClient, input *RepoApplyWrites_Input) (*RepoApplyWrites_Output, error) { + var out RepoApplyWrites_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.repo.applyWrites", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/repobatchWrite.go b/api/atproto/repobatchWrite.go deleted file mode 100644 index 77c429f7b..000000000 --- a/api/atproto/repobatchWrite.go +++ /dev/null @@ -1,96 +0,0 @@ -package schemagen - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.repo.batchWrite - -func init() { -} - -type RepoBatchWrite_Create struct { - LexiconTypeID string `json:"$type,omitempty"` - Action string `json:"action" cborgen:"action"` - Collection string `json:"collection" cborgen:"collection"` - Rkey *string `json:"rkey,omitempty" cborgen:"rkey"` - Value util.LexiconTypeDecoder `json:"value" cborgen:"value"` -} - -type RepoBatchWrite_Delete struct { - LexiconTypeID string `json:"$type,omitempty"` - Action string `json:"action" cborgen:"action"` - Collection string `json:"collection" cborgen:"collection"` - Rkey string `json:"rkey" cborgen:"rkey"` -} - -type RepoBatchWrite_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Did string `json:"did" cborgen:"did"` - Validate *bool `json:"validate,omitempty" cborgen:"validate"` - Writes []*RepoBatchWrite_Input_Writes_Elem `json:"writes" cborgen:"writes"` -} - -type RepoBatchWrite_Input_Writes_Elem struct { - RepoBatchWrite_Create *RepoBatchWrite_Create - RepoBatchWrite_Update *RepoBatchWrite_Update - RepoBatchWrite_Delete *RepoBatchWrite_Delete -} - -func (t *RepoBatchWrite_Input_Writes_Elem) MarshalJSON() ([]byte, error) { - if t.RepoBatchWrite_Create != nil { - t.RepoBatchWrite_Create.LexiconTypeID = "com.atproto.repo.batchWrite#create" - return json.Marshal(t.RepoBatchWrite_Create) - } - if t.RepoBatchWrite_Update != nil { - t.RepoBatchWrite_Update.LexiconTypeID = "com.atproto.repo.batchWrite#update" - return json.Marshal(t.RepoBatchWrite_Update) - } - if t.RepoBatchWrite_Delete != nil { - t.RepoBatchWrite_Delete.LexiconTypeID = "com.atproto.repo.batchWrite#delete" - return json.Marshal(t.RepoBatchWrite_Delete) - } - return nil, fmt.Errorf("cannot marshal empty enum") -} -func (t *RepoBatchWrite_Input_Writes_Elem) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) - if err != nil { - return err - } - - switch typ { - case "com.atproto.repo.batchWrite#create": - t.RepoBatchWrite_Create = new(RepoBatchWrite_Create) - return json.Unmarshal(b, t.RepoBatchWrite_Create) - case "com.atproto.repo.batchWrite#update": - t.RepoBatchWrite_Update = new(RepoBatchWrite_Update) - return json.Unmarshal(b, t.RepoBatchWrite_Update) - case "com.atproto.repo.batchWrite#delete": - t.RepoBatchWrite_Delete = new(RepoBatchWrite_Delete) - return json.Unmarshal(b, t.RepoBatchWrite_Delete) - - default: - return fmt.Errorf("closed enums must have a matching value") - } -} - -type RepoBatchWrite_Update struct { - LexiconTypeID string `json:"$type,omitempty"` - Action string `json:"action" cborgen:"action"` - Collection string `json:"collection" cborgen:"collection"` - Rkey string `json:"rkey" cborgen:"rkey"` - Value util.LexiconTypeDecoder `json:"value" cborgen:"value"` -} - -func RepoBatchWrite(ctx context.Context, c *xrpc.Client, input *RepoBatchWrite_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.batchWrite", nil, input, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/repocreateRecord.go b/api/atproto/repocreateRecord.go index 9c0f2191d..fffcf8e04 100644 --- a/api/atproto/repocreateRecord.go +++ b/api/atproto/repocreateRecord.go @@ -1,34 +1,43 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.createRecord + +package atproto import ( "context" - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: com.atproto.repo.createRecord - -func init() { -} - +// RepoCreateRecord_Input is the input argument to a com.atproto.repo.createRecord call. type RepoCreateRecord_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Collection string `json:"collection" cborgen:"collection"` - Did string `json:"did" cborgen:"did"` - Record util.LexiconTypeDecoder `json:"record" cborgen:"record"` - Validate *bool `json:"validate,omitempty" cborgen:"validate"` + // collection: The NSID of the record collection. + Collection string `json:"collection" cborgen:"collection"` + // record: The record itself. Must contain a $type field. + Record *lexutil.LexiconTypeDecoder `json:"record" cborgen:"record"` + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // rkey: The Record Key. + Rkey *string `json:"rkey,omitempty" cborgen:"rkey,omitempty"` + // swapCommit: Compare and swap with the previous commit by CID. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // validate: Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons. + Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` } +// RepoCreateRecord_Output is the output of a com.atproto.repo.createRecord call. type RepoCreateRecord_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid string `json:"cid" cborgen:"cid"` - Uri string `json:"uri" cborgen:"uri"` + Cid string `json:"cid" cborgen:"cid"` + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` } -func RepoCreateRecord(ctx context.Context, c *xrpc.Client, input *RepoCreateRecord_Input) (*RepoCreateRecord_Output, error) { +// RepoCreateRecord calls the XRPC method "com.atproto.repo.createRecord". +func RepoCreateRecord(ctx context.Context, c lexutil.LexClient, input *RepoCreateRecord_Input) (*RepoCreateRecord_Output, error) { var out RepoCreateRecord_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.createRecord", nil, input, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.repo.createRecord", nil, input, &out); err != nil { return nil, err } diff --git a/api/atproto/repodefs.go b/api/atproto/repodefs.go new file mode 100644 index 000000000..1b3acbe9a --- /dev/null +++ b/api/atproto/repodefs.go @@ -0,0 +1,11 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.defs + +package atproto + +// RepoDefs_CommitMeta is a "commitMeta" in the com.atproto.repo.defs schema. +type RepoDefs_CommitMeta struct { + Cid string `json:"cid" cborgen:"cid"` + Rev string `json:"rev" cborgen:"rev"` +} diff --git a/api/atproto/repodeleteRecord.go b/api/atproto/repodeleteRecord.go index a191cab75..17d808c28 100644 --- a/api/atproto/repodeleteRecord.go +++ b/api/atproto/repodeleteRecord.go @@ -1,27 +1,40 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.deleteRecord + +package atproto import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: com.atproto.repo.deleteRecord - -func init() { +// RepoDeleteRecord_Input is the input argument to a com.atproto.repo.deleteRecord call. +type RepoDeleteRecord_Input struct { + // collection: The NSID of the record collection. + Collection string `json:"collection" cborgen:"collection"` + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // rkey: The Record Key. + Rkey string `json:"rkey" cborgen:"rkey"` + // swapCommit: Compare and swap with the previous commit by CID. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // swapRecord: Compare and swap with the previous record by CID. + SwapRecord *string `json:"swapRecord,omitempty" cborgen:"swapRecord,omitempty"` } -type RepoDeleteRecord_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Collection string `json:"collection" cborgen:"collection"` - Did string `json:"did" cborgen:"did"` - Rkey string `json:"rkey" cborgen:"rkey"` +// RepoDeleteRecord_Output is the output of a com.atproto.repo.deleteRecord call. +type RepoDeleteRecord_Output struct { + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` } -func RepoDeleteRecord(ctx context.Context, c *xrpc.Client, input *RepoDeleteRecord_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.deleteRecord", nil, input, nil); err != nil { - return err +// RepoDeleteRecord calls the XRPC method "com.atproto.repo.deleteRecord". +func RepoDeleteRecord(ctx context.Context, c lexutil.LexClient, input *RepoDeleteRecord_Input) (*RepoDeleteRecord_Output, error) { + var out RepoDeleteRecord_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.repo.deleteRecord", nil, input, &out); err != nil { + return nil, err } - return nil + return &out, nil } diff --git a/api/atproto/repodescribe.go b/api/atproto/repodescribe.go deleted file mode 100644 index f77f3a687..000000000 --- a/api/atproto/repodescribe.go +++ /dev/null @@ -1,35 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.repo.describe - -func init() { -} - -type RepoDescribe_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Collections []string `json:"collections" cborgen:"collections"` - Did string `json:"did" cborgen:"did"` - DidDoc util.LexiconTypeDecoder `json:"didDoc" cborgen:"didDoc"` - Handle string `json:"handle" cborgen:"handle"` - HandleIsCorrect bool `json:"handleIsCorrect" cborgen:"handleIsCorrect"` -} - -func RepoDescribe(ctx context.Context, c *xrpc.Client, user string) (*RepoDescribe_Output, error) { - var out RepoDescribe_Output - - params := map[string]interface{}{ - "user": user, - } - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.repo.describe", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/repodescribeRepo.go b/api/atproto/repodescribeRepo.go new file mode 100644 index 000000000..178fd94c9 --- /dev/null +++ b/api/atproto/repodescribeRepo.go @@ -0,0 +1,38 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.describeRepo + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// RepoDescribeRepo_Output is the output of a com.atproto.repo.describeRepo call. +type RepoDescribeRepo_Output struct { + // collections: List of all the collections (NSIDs) for which this repo contains at least one record. + Collections []string `json:"collections" cborgen:"collections"` + Did string `json:"did" cborgen:"did"` + // didDoc: The complete DID document for this account. + DidDoc interface{} `json:"didDoc" cborgen:"didDoc"` + Handle string `json:"handle" cborgen:"handle"` + // handleIsCorrect: Indicates if handle is currently valid (resolves bi-directionally) + HandleIsCorrect bool `json:"handleIsCorrect" cborgen:"handleIsCorrect"` +} + +// RepoDescribeRepo calls the XRPC method "com.atproto.repo.describeRepo". +// +// repo: The handle or DID of the repo. +func RepoDescribeRepo(ctx context.Context, c lexutil.LexClient, repo string) (*RepoDescribeRepo_Output, error) { + var out RepoDescribeRepo_Output + + params := map[string]interface{}{} + params["repo"] = repo + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.repo.describeRepo", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/repogetRecord.go b/api/atproto/repogetRecord.go index b7cc9041a..5a40418e0 100644 --- a/api/atproto/repogetRecord.go +++ b/api/atproto/repogetRecord.go @@ -1,34 +1,39 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.getRecord + +package atproto import ( "context" - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: com.atproto.repo.getRecord - -func init() { -} - +// RepoGetRecord_Output is the output of a com.atproto.repo.getRecord call. type RepoGetRecord_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid *string `json:"cid,omitempty" cborgen:"cid"` - Uri string `json:"uri" cborgen:"uri"` - Value util.LexiconTypeDecoder `json:"value" cborgen:"value"` + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` } -func RepoGetRecord(ctx context.Context, c *xrpc.Client, cid string, collection string, rkey string, user string) (*RepoGetRecord_Output, error) { +// RepoGetRecord calls the XRPC method "com.atproto.repo.getRecord". +// +// cid: The CID of the version of the record. If not specified, then return the most recent version. +// collection: The NSID of the record collection. +// repo: The handle or DID of the repo. +// rkey: The Record Key. +func RepoGetRecord(ctx context.Context, c lexutil.LexClient, cid string, collection string, repo string, rkey string) (*RepoGetRecord_Output, error) { var out RepoGetRecord_Output - params := map[string]interface{}{ - "cid": cid, - "collection": collection, - "rkey": rkey, - "user": user, + params := map[string]interface{}{} + if cid != "" { + params["cid"] = cid } - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.repo.getRecord", params, nil, &out); err != nil { + params["collection"] = collection + params["repo"] = repo + params["rkey"] = rkey + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.repo.getRecord", params, nil, &out); err != nil { return nil, err } diff --git a/api/atproto/repoimportRepo.go b/api/atproto/repoimportRepo.go new file mode 100644 index 000000000..8bb165745 --- /dev/null +++ b/api/atproto/repoimportRepo.go @@ -0,0 +1,21 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.importRepo + +package atproto + +import ( + "context" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// RepoImportRepo calls the XRPC method "com.atproto.repo.importRepo". +func RepoImportRepo(ctx context.Context, c lexutil.LexClient, input io.Reader) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/vnd.ipld.car", "com.atproto.repo.importRepo", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/repolistMissingBlobs.go b/api/atproto/repolistMissingBlobs.go new file mode 100644 index 000000000..2cfaf0e74 --- /dev/null +++ b/api/atproto/repolistMissingBlobs.go @@ -0,0 +1,41 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.listMissingBlobs + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// RepoListMissingBlobs_Output is the output of a com.atproto.repo.listMissingBlobs call. +type RepoListMissingBlobs_Output struct { + Blobs []*RepoListMissingBlobs_RecordBlob `json:"blobs" cborgen:"blobs"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// RepoListMissingBlobs_RecordBlob is a "recordBlob" in the com.atproto.repo.listMissingBlobs schema. +type RepoListMissingBlobs_RecordBlob struct { + Cid string `json:"cid" cborgen:"cid"` + RecordUri string `json:"recordUri" cborgen:"recordUri"` +} + +// RepoListMissingBlobs calls the XRPC method "com.atproto.repo.listMissingBlobs". +func RepoListMissingBlobs(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*RepoListMissingBlobs_Output, error) { + var out RepoListMissingBlobs_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.repo.listMissingBlobs", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/repolistRecords.go b/api/atproto/repolistRecords.go index a076d9413..9e6cabc26 100644 --- a/api/atproto/repolistRecords.go +++ b/api/atproto/repolistRecords.go @@ -1,42 +1,50 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.listRecords + +package atproto import ( "context" - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: com.atproto.repo.listRecords - -func init() { -} - +// RepoListRecords_Output is the output of a com.atproto.repo.listRecords call. type RepoListRecords_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Records []*RepoListRecords_Record `json:"records" cborgen:"records"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Records []*RepoListRecords_Record `json:"records" cborgen:"records"` } +// RepoListRecords_Record is a "record" in the com.atproto.repo.listRecords schema. type RepoListRecords_Record struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid string `json:"cid" cborgen:"cid"` - Uri string `json:"uri" cborgen:"uri"` - Value util.LexiconTypeDecoder `json:"value" cborgen:"value"` + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` } -func RepoListRecords(ctx context.Context, c *xrpc.Client, after string, before string, collection string, limit int64, reverse bool, user string) (*RepoListRecords_Output, error) { +// RepoListRecords calls the XRPC method "com.atproto.repo.listRecords". +// +// collection: The NSID of the record type. +// limit: The number of records to return. +// repo: The handle or DID of the repo. +// reverse: Flag to reverse the order of the returned records. +func RepoListRecords(ctx context.Context, c lexutil.LexClient, collection string, cursor string, limit int64, repo string, reverse bool) (*RepoListRecords_Output, error) { var out RepoListRecords_Output - params := map[string]interface{}{ - "after": after, - "before": before, - "collection": collection, - "limit": limit, - "reverse": reverse, - "user": user, + params := map[string]interface{}{} + params["collection"] = collection + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["repo"] = repo + if reverse { + params["reverse"] = reverse } - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.repo.listRecords", params, nil, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.repo.listRecords", params, nil, &out); err != nil { return nil, err } diff --git a/api/atproto/repoputRecord.go b/api/atproto/repoputRecord.go index 7d68e75fc..091a95db4 100644 --- a/api/atproto/repoputRecord.go +++ b/api/atproto/repoputRecord.go @@ -1,35 +1,45 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.putRecord + +package atproto import ( "context" - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: com.atproto.repo.putRecord - -func init() { -} - +// RepoPutRecord_Input is the input argument to a com.atproto.repo.putRecord call. type RepoPutRecord_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Collection string `json:"collection" cborgen:"collection"` - Did string `json:"did" cborgen:"did"` - Record util.LexiconTypeDecoder `json:"record" cborgen:"record"` - Rkey string `json:"rkey" cborgen:"rkey"` - Validate *bool `json:"validate,omitempty" cborgen:"validate"` + // collection: The NSID of the record collection. + Collection string `json:"collection" cborgen:"collection"` + // record: The record to write. + Record *lexutil.LexiconTypeDecoder `json:"record" cborgen:"record"` + // repo: The handle or DID of the repo (aka, current account). + Repo string `json:"repo" cborgen:"repo"` + // rkey: The Record Key. + Rkey string `json:"rkey" cborgen:"rkey"` + // swapCommit: Compare and swap with the previous commit by CID. + SwapCommit *string `json:"swapCommit,omitempty" cborgen:"swapCommit,omitempty"` + // swapRecord: Compare and swap with the previous record by CID. WARNING: nullable and optional field; may cause problems with golang implementation + SwapRecord *string `json:"swapRecord" cborgen:"swapRecord"` + // validate: Can be set to 'false' to skip Lexicon schema validation of record data, 'true' to require it, or leave unset to validate only for known Lexicons. + Validate *bool `json:"validate,omitempty" cborgen:"validate,omitempty"` } +// RepoPutRecord_Output is the output of a com.atproto.repo.putRecord call. type RepoPutRecord_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid string `json:"cid" cborgen:"cid"` - Uri string `json:"uri" cborgen:"uri"` + Cid string `json:"cid" cborgen:"cid"` + Commit *RepoDefs_CommitMeta `json:"commit,omitempty" cborgen:"commit,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + ValidationStatus *string `json:"validationStatus,omitempty" cborgen:"validationStatus,omitempty"` } -func RepoPutRecord(ctx context.Context, c *xrpc.Client, input *RepoPutRecord_Input) (*RepoPutRecord_Output, error) { +// RepoPutRecord calls the XRPC method "com.atproto.repo.putRecord". +func RepoPutRecord(ctx context.Context, c lexutil.LexClient, input *RepoPutRecord_Input) (*RepoPutRecord_Output, error) { var out RepoPutRecord_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.repo.putRecord", nil, input, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.repo.putRecord", nil, input, &out); err != nil { return nil, err } diff --git a/api/atproto/repostrongRef.go b/api/atproto/repostrongRef.go index d65583c1f..fd465d71d 100644 --- a/api/atproto/repostrongRef.go +++ b/api/atproto/repostrongRef.go @@ -1,12 +1,20 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. -// schema: com.atproto.repo.strongRef +// Lexicon schema: com.atproto.repo.strongRef + +package atproto + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) func init() { + lexutil.RegisterType("com.atproto.repo.strongRef#main", &RepoStrongRef{}) } +// RepoStrongRef is a "main" in the com.atproto.repo.strongRef schema. type RepoStrongRef struct { - LexiconTypeID string `json:"$type,omitempty"` + LexiconTypeID string `json:"$type,omitempty" cborgen:"$type,const=com.atproto.repo.strongRef,omitempty"` Cid string `json:"cid" cborgen:"cid"` Uri string `json:"uri" cborgen:"uri"` } diff --git a/api/atproto/repouploadBlob.go b/api/atproto/repouploadBlob.go new file mode 100644 index 000000000..40deb9491 --- /dev/null +++ b/api/atproto/repouploadBlob.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.repo.uploadBlob + +package atproto + +import ( + "context" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// RepoUploadBlob_Output is the output of a com.atproto.repo.uploadBlob call. +type RepoUploadBlob_Output struct { + Blob *lexutil.LexBlob `json:"blob" cborgen:"blob"` +} + +// RepoUploadBlob calls the XRPC method "com.atproto.repo.uploadBlob". +func RepoUploadBlob(ctx context.Context, c lexutil.LexClient, input io.Reader) (*RepoUploadBlob_Output, error) { + var out RepoUploadBlob_Output + if err := c.LexDo(ctx, lexutil.Procedure, "*/*", "com.atproto.repo.uploadBlob", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serveractivateAccount.go b/api/atproto/serveractivateAccount.go new file mode 100644 index 000000000..ad0344727 --- /dev/null +++ b/api/atproto/serveractivateAccount.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.activateAccount + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerActivateAccount calls the XRPC method "com.atproto.server.activateAccount". +func ServerActivateAccount(ctx context.Context, c lexutil.LexClient) error { + if err := c.LexDo(ctx, lexutil.Procedure, "", "com.atproto.server.activateAccount", nil, nil, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/servercheckAccountStatus.go b/api/atproto/servercheckAccountStatus.go new file mode 100644 index 000000000..f307d1888 --- /dev/null +++ b/api/atproto/servercheckAccountStatus.go @@ -0,0 +1,34 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.checkAccountStatus + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerCheckAccountStatus_Output is the output of a com.atproto.server.checkAccountStatus call. +type ServerCheckAccountStatus_Output struct { + Activated bool `json:"activated" cborgen:"activated"` + ExpectedBlobs int64 `json:"expectedBlobs" cborgen:"expectedBlobs"` + ImportedBlobs int64 `json:"importedBlobs" cborgen:"importedBlobs"` + IndexedRecords int64 `json:"indexedRecords" cborgen:"indexedRecords"` + PrivateStateValues int64 `json:"privateStateValues" cborgen:"privateStateValues"` + RepoBlocks int64 `json:"repoBlocks" cborgen:"repoBlocks"` + RepoCommit string `json:"repoCommit" cborgen:"repoCommit"` + RepoRev string `json:"repoRev" cborgen:"repoRev"` + ValidDid bool `json:"validDid" cborgen:"validDid"` +} + +// ServerCheckAccountStatus calls the XRPC method "com.atproto.server.checkAccountStatus". +func ServerCheckAccountStatus(ctx context.Context, c lexutil.LexClient) (*ServerCheckAccountStatus_Output, error) { + var out ServerCheckAccountStatus_Output + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.server.checkAccountStatus", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serverconfirmEmail.go b/api/atproto/serverconfirmEmail.go new file mode 100644 index 000000000..aa7707356 --- /dev/null +++ b/api/atproto/serverconfirmEmail.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.confirmEmail + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerConfirmEmail_Input is the input argument to a com.atproto.server.confirmEmail call. +type ServerConfirmEmail_Input struct { + Email string `json:"email" cborgen:"email"` + Token string `json:"token" cborgen:"token"` +} + +// ServerConfirmEmail calls the XRPC method "com.atproto.server.confirmEmail". +func ServerConfirmEmail(ctx context.Context, c lexutil.LexClient, input *ServerConfirmEmail_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.confirmEmail", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/servercreateAccount.go b/api/atproto/servercreateAccount.go new file mode 100644 index 000000000..8589beeac --- /dev/null +++ b/api/atproto/servercreateAccount.go @@ -0,0 +1,52 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.createAccount + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerCreateAccount_Input is the input argument to a com.atproto.server.createAccount call. +type ServerCreateAccount_Input struct { + // did: Pre-existing atproto DID, being imported to a new account. + Did *string `json:"did,omitempty" cborgen:"did,omitempty"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + // handle: Requested handle for the account. + Handle string `json:"handle" cborgen:"handle"` + InviteCode *string `json:"inviteCode,omitempty" cborgen:"inviteCode,omitempty"` + // password: Initial account password. May need to meet instance-specific password strength requirements. + Password *string `json:"password,omitempty" cborgen:"password,omitempty"` + // plcOp: A signed DID PLC operation to be submitted as part of importing an existing account to this instance. NOTE: this optional field may be updated when full account migration is implemented. + PlcOp *interface{} `json:"plcOp,omitempty" cborgen:"plcOp,omitempty"` + // recoveryKey: DID PLC rotation key (aka, recovery key) to be included in PLC creation operation. + RecoveryKey *string `json:"recoveryKey,omitempty" cborgen:"recoveryKey,omitempty"` + VerificationCode *string `json:"verificationCode,omitempty" cborgen:"verificationCode,omitempty"` + VerificationPhone *string `json:"verificationPhone,omitempty" cborgen:"verificationPhone,omitempty"` +} + +// ServerCreateAccount_Output is the output of a com.atproto.server.createAccount call. +// +// Account login session returned on successful account creation. +type ServerCreateAccount_Output struct { + AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` + // did: The DID of the new account. + Did string `json:"did" cborgen:"did"` + // didDoc: Complete DID document. + DidDoc *interface{} `json:"didDoc,omitempty" cborgen:"didDoc,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` +} + +// ServerCreateAccount calls the XRPC method "com.atproto.server.createAccount". +func ServerCreateAccount(ctx context.Context, c lexutil.LexClient, input *ServerCreateAccount_Input) (*ServerCreateAccount_Output, error) { + var out ServerCreateAccount_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.createAccount", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/servercreateAppPassword.go b/api/atproto/servercreateAppPassword.go new file mode 100644 index 000000000..456231f74 --- /dev/null +++ b/api/atproto/servercreateAppPassword.go @@ -0,0 +1,37 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.createAppPassword + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerCreateAppPassword_AppPassword is a "appPassword" in the com.atproto.server.createAppPassword schema. +type ServerCreateAppPassword_AppPassword struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Name string `json:"name" cborgen:"name"` + Password string `json:"password" cborgen:"password"` + Privileged *bool `json:"privileged,omitempty" cborgen:"privileged,omitempty"` +} + +// ServerCreateAppPassword_Input is the input argument to a com.atproto.server.createAppPassword call. +type ServerCreateAppPassword_Input struct { + // name: A short name for the App Password, to help distinguish them. + Name string `json:"name" cborgen:"name"` + // privileged: If an app password has 'privileged' access to possibly sensitive account state. Meant for use with trusted clients. + Privileged *bool `json:"privileged,omitempty" cborgen:"privileged,omitempty"` +} + +// ServerCreateAppPassword calls the XRPC method "com.atproto.server.createAppPassword". +func ServerCreateAppPassword(ctx context.Context, c lexutil.LexClient, input *ServerCreateAppPassword_Input) (*ServerCreateAppPassword_AppPassword, error) { + var out ServerCreateAppPassword_AppPassword + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.createAppPassword", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/servercreateInviteCode.go b/api/atproto/servercreateInviteCode.go new file mode 100644 index 000000000..84759b0a2 --- /dev/null +++ b/api/atproto/servercreateInviteCode.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.createInviteCode + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerCreateInviteCode_Input is the input argument to a com.atproto.server.createInviteCode call. +type ServerCreateInviteCode_Input struct { + ForAccount *string `json:"forAccount,omitempty" cborgen:"forAccount,omitempty"` + UseCount int64 `json:"useCount" cborgen:"useCount"` +} + +// ServerCreateInviteCode_Output is the output of a com.atproto.server.createInviteCode call. +type ServerCreateInviteCode_Output struct { + Code string `json:"code" cborgen:"code"` +} + +// ServerCreateInviteCode calls the XRPC method "com.atproto.server.createInviteCode". +func ServerCreateInviteCode(ctx context.Context, c lexutil.LexClient, input *ServerCreateInviteCode_Input) (*ServerCreateInviteCode_Output, error) { + var out ServerCreateInviteCode_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.createInviteCode", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/servercreateInviteCodes.go b/api/atproto/servercreateInviteCodes.go new file mode 100644 index 000000000..649ac203d --- /dev/null +++ b/api/atproto/servercreateInviteCodes.go @@ -0,0 +1,39 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.createInviteCodes + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerCreateInviteCodes_AccountCodes is a "accountCodes" in the com.atproto.server.createInviteCodes schema. +type ServerCreateInviteCodes_AccountCodes struct { + Account string `json:"account" cborgen:"account"` + Codes []string `json:"codes" cborgen:"codes"` +} + +// ServerCreateInviteCodes_Input is the input argument to a com.atproto.server.createInviteCodes call. +type ServerCreateInviteCodes_Input struct { + CodeCount int64 `json:"codeCount" cborgen:"codeCount"` + ForAccounts []string `json:"forAccounts,omitempty" cborgen:"forAccounts,omitempty"` + UseCount int64 `json:"useCount" cborgen:"useCount"` +} + +// ServerCreateInviteCodes_Output is the output of a com.atproto.server.createInviteCodes call. +type ServerCreateInviteCodes_Output struct { + Codes []*ServerCreateInviteCodes_AccountCodes `json:"codes" cborgen:"codes"` +} + +// ServerCreateInviteCodes calls the XRPC method "com.atproto.server.createInviteCodes". +func ServerCreateInviteCodes(ctx context.Context, c lexutil.LexClient, input *ServerCreateInviteCodes_Input) (*ServerCreateInviteCodes_Output, error) { + var out ServerCreateInviteCodes_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.createInviteCodes", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/servercreateSession.go b/api/atproto/servercreateSession.go new file mode 100644 index 000000000..bd508f9da --- /dev/null +++ b/api/atproto/servercreateSession.go @@ -0,0 +1,46 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.createSession + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerCreateSession_Input is the input argument to a com.atproto.server.createSession call. +type ServerCreateSession_Input struct { + // allowTakendown: When true, instead of throwing error for takendown accounts, a valid response with a narrow scoped token will be returned + AllowTakendown *bool `json:"allowTakendown,omitempty" cborgen:"allowTakendown,omitempty"` + AuthFactorToken *string `json:"authFactorToken,omitempty" cborgen:"authFactorToken,omitempty"` + // identifier: Handle or other identifier supported by the server for the authenticating user. + Identifier string `json:"identifier" cborgen:"identifier"` + Password string `json:"password" cborgen:"password"` +} + +// ServerCreateSession_Output is the output of a com.atproto.server.createSession call. +type ServerCreateSession_Output struct { + AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` + Active *bool `json:"active,omitempty" cborgen:"active,omitempty"` + Did string `json:"did" cborgen:"did"` + DidDoc *interface{} `json:"didDoc,omitempty" cborgen:"didDoc,omitempty"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailAuthFactor *bool `json:"emailAuthFactor,omitempty" cborgen:"emailAuthFactor,omitempty"` + EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` + // status: If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted. + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +// ServerCreateSession calls the XRPC method "com.atproto.server.createSession". +func ServerCreateSession(ctx context.Context, c lexutil.LexClient, input *ServerCreateSession_Input) (*ServerCreateSession_Output, error) { + var out ServerCreateSession_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.createSession", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serverdeactivateAccount.go b/api/atproto/serverdeactivateAccount.go new file mode 100644 index 000000000..f48e63882 --- /dev/null +++ b/api/atproto/serverdeactivateAccount.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.deactivateAccount + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerDeactivateAccount_Input is the input argument to a com.atproto.server.deactivateAccount call. +type ServerDeactivateAccount_Input struct { + // deleteAfter: A recommendation to server as to how long they should hold onto the deactivated account before deleting. + DeleteAfter *string `json:"deleteAfter,omitempty" cborgen:"deleteAfter,omitempty"` +} + +// ServerDeactivateAccount calls the XRPC method "com.atproto.server.deactivateAccount". +func ServerDeactivateAccount(ctx context.Context, c lexutil.LexClient, input *ServerDeactivateAccount_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.deactivateAccount", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverdefs.go b/api/atproto/serverdefs.go new file mode 100644 index 000000000..03ef5fda6 --- /dev/null +++ b/api/atproto/serverdefs.go @@ -0,0 +1,22 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.defs + +package atproto + +// ServerDefs_InviteCode is a "inviteCode" in the com.atproto.server.defs schema. +type ServerDefs_InviteCode struct { + Available int64 `json:"available" cborgen:"available"` + Code string `json:"code" cborgen:"code"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + Disabled bool `json:"disabled" cborgen:"disabled"` + ForAccount string `json:"forAccount" cborgen:"forAccount"` + Uses []*ServerDefs_InviteCodeUse `json:"uses" cborgen:"uses"` +} + +// ServerDefs_InviteCodeUse is a "inviteCodeUse" in the com.atproto.server.defs schema. +type ServerDefs_InviteCodeUse struct { + UsedAt string `json:"usedAt" cborgen:"usedAt"` + UsedBy string `json:"usedBy" cborgen:"usedBy"` +} diff --git a/api/atproto/serverdeleteAccount.go b/api/atproto/serverdeleteAccount.go new file mode 100644 index 000000000..ce4e57d84 --- /dev/null +++ b/api/atproto/serverdeleteAccount.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.deleteAccount + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerDeleteAccount_Input is the input argument to a com.atproto.server.deleteAccount call. +type ServerDeleteAccount_Input struct { + Did string `json:"did" cborgen:"did"` + Password string `json:"password" cborgen:"password"` + Token string `json:"token" cborgen:"token"` +} + +// ServerDeleteAccount calls the XRPC method "com.atproto.server.deleteAccount". +func ServerDeleteAccount(ctx context.Context, c lexutil.LexClient, input *ServerDeleteAccount_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.deleteAccount", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverdeleteSession.go b/api/atproto/serverdeleteSession.go new file mode 100644 index 000000000..adce5390e --- /dev/null +++ b/api/atproto/serverdeleteSession.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.deleteSession + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerDeleteSession calls the XRPC method "com.atproto.server.deleteSession". +func ServerDeleteSession(ctx context.Context, c lexutil.LexClient) error { + if err := c.LexDo(ctx, lexutil.Procedure, "", "com.atproto.server.deleteSession", nil, nil, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverdescribeServer.go b/api/atproto/serverdescribeServer.go new file mode 100644 index 000000000..76679c0dc --- /dev/null +++ b/api/atproto/serverdescribeServer.go @@ -0,0 +1,47 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.describeServer + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerDescribeServer_Contact is a "contact" in the com.atproto.server.describeServer schema. +type ServerDescribeServer_Contact struct { + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` +} + +// ServerDescribeServer_Links is a "links" in the com.atproto.server.describeServer schema. +type ServerDescribeServer_Links struct { + PrivacyPolicy *string `json:"privacyPolicy,omitempty" cborgen:"privacyPolicy,omitempty"` + TermsOfService *string `json:"termsOfService,omitempty" cborgen:"termsOfService,omitempty"` +} + +// ServerDescribeServer_Output is the output of a com.atproto.server.describeServer call. +type ServerDescribeServer_Output struct { + // availableUserDomains: List of domain suffixes that can be used in account handles. + AvailableUserDomains []string `json:"availableUserDomains" cborgen:"availableUserDomains"` + // contact: Contact information + Contact *ServerDescribeServer_Contact `json:"contact,omitempty" cborgen:"contact,omitempty"` + Did string `json:"did" cborgen:"did"` + // inviteCodeRequired: If true, an invite code must be supplied to create an account on this instance. + InviteCodeRequired *bool `json:"inviteCodeRequired,omitempty" cborgen:"inviteCodeRequired,omitempty"` + // links: URLs of service policy documents. + Links *ServerDescribeServer_Links `json:"links,omitempty" cborgen:"links,omitempty"` + // phoneVerificationRequired: If true, a phone verification token must be supplied to create an account on this instance. + PhoneVerificationRequired *bool `json:"phoneVerificationRequired,omitempty" cborgen:"phoneVerificationRequired,omitempty"` +} + +// ServerDescribeServer calls the XRPC method "com.atproto.server.describeServer". +func ServerDescribeServer(ctx context.Context, c lexutil.LexClient) (*ServerDescribeServer_Output, error) { + var out ServerDescribeServer_Output + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.server.describeServer", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/servergetAccountInviteCodes.go b/api/atproto/servergetAccountInviteCodes.go new file mode 100644 index 000000000..8841e762f --- /dev/null +++ b/api/atproto/servergetAccountInviteCodes.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.getAccountInviteCodes + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerGetAccountInviteCodes_Output is the output of a com.atproto.server.getAccountInviteCodes call. +type ServerGetAccountInviteCodes_Output struct { + Codes []*ServerDefs_InviteCode `json:"codes" cborgen:"codes"` +} + +// ServerGetAccountInviteCodes calls the XRPC method "com.atproto.server.getAccountInviteCodes". +// +// createAvailable: Controls whether any new 'earned' but not 'created' invites should be created. +func ServerGetAccountInviteCodes(ctx context.Context, c lexutil.LexClient, createAvailable bool, includeUsed bool) (*ServerGetAccountInviteCodes_Output, error) { + var out ServerGetAccountInviteCodes_Output + + params := map[string]interface{}{} + if createAvailable { + params["createAvailable"] = createAvailable + } + if includeUsed { + params["includeUsed"] = includeUsed + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.server.getAccountInviteCodes", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/servergetAccountsConfig.go b/api/atproto/servergetAccountsConfig.go deleted file mode 100644 index 5f31a5a39..000000000 --- a/api/atproto/servergetAccountsConfig.go +++ /dev/null @@ -1,34 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.server.getAccountsConfig - -func init() { -} - -type ServerGetAccountsConfig_Links struct { - LexiconTypeID string `json:"$type,omitempty"` - PrivacyPolicy *string `json:"privacyPolicy,omitempty" cborgen:"privacyPolicy"` - TermsOfService *string `json:"termsOfService,omitempty" cborgen:"termsOfService"` -} - -type ServerGetAccountsConfig_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - AvailableUserDomains []string `json:"availableUserDomains" cborgen:"availableUserDomains"` - InviteCodeRequired *bool `json:"inviteCodeRequired,omitempty" cborgen:"inviteCodeRequired"` - Links *ServerGetAccountsConfig_Links `json:"links,omitempty" cborgen:"links"` -} - -func ServerGetAccountsConfig(ctx context.Context, c *xrpc.Client) (*ServerGetAccountsConfig_Output, error) { - var out ServerGetAccountsConfig_Output - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.server.getAccountsConfig", nil, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/servergetServiceAuth.go b/api/atproto/servergetServiceAuth.go new file mode 100644 index 000000000..052744da3 --- /dev/null +++ b/api/atproto/servergetServiceAuth.go @@ -0,0 +1,39 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.getServiceAuth + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerGetServiceAuth_Output is the output of a com.atproto.server.getServiceAuth call. +type ServerGetServiceAuth_Output struct { + Token string `json:"token" cborgen:"token"` +} + +// ServerGetServiceAuth calls the XRPC method "com.atproto.server.getServiceAuth". +// +// aud: The DID of the service that the token will be used to authenticate with +// exp: The time in Unix Epoch seconds that the JWT expires. Defaults to 60 seconds in the future. The service may enforce certain time bounds on tokens depending on the requested scope. +// lxm: Lexicon (XRPC) method to bind the requested token to +func ServerGetServiceAuth(ctx context.Context, c lexutil.LexClient, aud string, exp int64, lxm string) (*ServerGetServiceAuth_Output, error) { + var out ServerGetServiceAuth_Output + + params := map[string]interface{}{} + params["aud"] = aud + if exp != 0 { + params["exp"] = exp + } + if lxm != "" { + params["lxm"] = lxm + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.server.getServiceAuth", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/servergetSession.go b/api/atproto/servergetSession.go new file mode 100644 index 000000000..323a3189f --- /dev/null +++ b/api/atproto/servergetSession.go @@ -0,0 +1,34 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.getSession + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerGetSession_Output is the output of a com.atproto.server.getSession call. +type ServerGetSession_Output struct { + Active *bool `json:"active,omitempty" cborgen:"active,omitempty"` + Did string `json:"did" cborgen:"did"` + DidDoc *interface{} `json:"didDoc,omitempty" cborgen:"didDoc,omitempty"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailAuthFactor *bool `json:"emailAuthFactor,omitempty" cborgen:"emailAuthFactor,omitempty"` + EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + // status: If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted. + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +// ServerGetSession calls the XRPC method "com.atproto.server.getSession". +func ServerGetSession(ctx context.Context, c lexutil.LexClient) (*ServerGetSession_Output, error) { + var out ServerGetSession_Output + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.server.getSession", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serverlistAppPasswords.go b/api/atproto/serverlistAppPasswords.go new file mode 100644 index 000000000..e5b000b27 --- /dev/null +++ b/api/atproto/serverlistAppPasswords.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.listAppPasswords + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerListAppPasswords_AppPassword is a "appPassword" in the com.atproto.server.listAppPasswords schema. +type ServerListAppPasswords_AppPassword struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Name string `json:"name" cborgen:"name"` + Privileged *bool `json:"privileged,omitempty" cborgen:"privileged,omitempty"` +} + +// ServerListAppPasswords_Output is the output of a com.atproto.server.listAppPasswords call. +type ServerListAppPasswords_Output struct { + Passwords []*ServerListAppPasswords_AppPassword `json:"passwords" cborgen:"passwords"` +} + +// ServerListAppPasswords calls the XRPC method "com.atproto.server.listAppPasswords". +func ServerListAppPasswords(ctx context.Context, c lexutil.LexClient) (*ServerListAppPasswords_Output, error) { + var out ServerListAppPasswords_Output + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.server.listAppPasswords", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serverrefreshSession.go b/api/atproto/serverrefreshSession.go new file mode 100644 index 000000000..a0a7d91a2 --- /dev/null +++ b/api/atproto/serverrefreshSession.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.refreshSession + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerRefreshSession_Output is the output of a com.atproto.server.refreshSession call. +type ServerRefreshSession_Output struct { + AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` + Active *bool `json:"active,omitempty" cborgen:"active,omitempty"` + Did string `json:"did" cborgen:"did"` + DidDoc *interface{} `json:"didDoc,omitempty" cborgen:"didDoc,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` + // status: Hosting status of the account. If not specified, then assume 'active'. + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +// ServerRefreshSession calls the XRPC method "com.atproto.server.refreshSession". +func ServerRefreshSession(ctx context.Context, c lexutil.LexClient) (*ServerRefreshSession_Output, error) { + var out ServerRefreshSession_Output + if err := c.LexDo(ctx, lexutil.Procedure, "", "com.atproto.server.refreshSession", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serverrequestAccountDelete.go b/api/atproto/serverrequestAccountDelete.go new file mode 100644 index 000000000..086fb140a --- /dev/null +++ b/api/atproto/serverrequestAccountDelete.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.requestAccountDelete + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerRequestAccountDelete calls the XRPC method "com.atproto.server.requestAccountDelete". +func ServerRequestAccountDelete(ctx context.Context, c lexutil.LexClient) error { + if err := c.LexDo(ctx, lexutil.Procedure, "", "com.atproto.server.requestAccountDelete", nil, nil, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverrequestEmailConfirmation.go b/api/atproto/serverrequestEmailConfirmation.go new file mode 100644 index 000000000..e96fbd66f --- /dev/null +++ b/api/atproto/serverrequestEmailConfirmation.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.requestEmailConfirmation + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerRequestEmailConfirmation calls the XRPC method "com.atproto.server.requestEmailConfirmation". +func ServerRequestEmailConfirmation(ctx context.Context, c lexutil.LexClient) error { + if err := c.LexDo(ctx, lexutil.Procedure, "", "com.atproto.server.requestEmailConfirmation", nil, nil, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverrequestEmailUpdate.go b/api/atproto/serverrequestEmailUpdate.go new file mode 100644 index 000000000..e3447acba --- /dev/null +++ b/api/atproto/serverrequestEmailUpdate.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.requestEmailUpdate + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerRequestEmailUpdate_Output is the output of a com.atproto.server.requestEmailUpdate call. +type ServerRequestEmailUpdate_Output struct { + TokenRequired bool `json:"tokenRequired" cborgen:"tokenRequired"` +} + +// ServerRequestEmailUpdate calls the XRPC method "com.atproto.server.requestEmailUpdate". +func ServerRequestEmailUpdate(ctx context.Context, c lexutil.LexClient) (*ServerRequestEmailUpdate_Output, error) { + var out ServerRequestEmailUpdate_Output + if err := c.LexDo(ctx, lexutil.Procedure, "", "com.atproto.server.requestEmailUpdate", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serverrequestPasswordReset.go b/api/atproto/serverrequestPasswordReset.go new file mode 100644 index 000000000..3a81e01de --- /dev/null +++ b/api/atproto/serverrequestPasswordReset.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.requestPasswordReset + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerRequestPasswordReset_Input is the input argument to a com.atproto.server.requestPasswordReset call. +type ServerRequestPasswordReset_Input struct { + Email string `json:"email" cborgen:"email"` +} + +// ServerRequestPasswordReset calls the XRPC method "com.atproto.server.requestPasswordReset". +func ServerRequestPasswordReset(ctx context.Context, c lexutil.LexClient, input *ServerRequestPasswordReset_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.requestPasswordReset", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverreserveSigningKey.go b/api/atproto/serverreserveSigningKey.go new file mode 100644 index 000000000..b24694484 --- /dev/null +++ b/api/atproto/serverreserveSigningKey.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.reserveSigningKey + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerReserveSigningKey_Input is the input argument to a com.atproto.server.reserveSigningKey call. +type ServerReserveSigningKey_Input struct { + // did: The DID to reserve a key for. + Did *string `json:"did,omitempty" cborgen:"did,omitempty"` +} + +// ServerReserveSigningKey_Output is the output of a com.atproto.server.reserveSigningKey call. +type ServerReserveSigningKey_Output struct { + // signingKey: The public key for the reserved signing key, in did:key serialization. + SigningKey string `json:"signingKey" cborgen:"signingKey"` +} + +// ServerReserveSigningKey calls the XRPC method "com.atproto.server.reserveSigningKey". +func ServerReserveSigningKey(ctx context.Context, c lexutil.LexClient, input *ServerReserveSigningKey_Input) (*ServerReserveSigningKey_Output, error) { + var out ServerReserveSigningKey_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.reserveSigningKey", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serverresetPassword.go b/api/atproto/serverresetPassword.go new file mode 100644 index 000000000..3037fd829 --- /dev/null +++ b/api/atproto/serverresetPassword.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.resetPassword + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerResetPassword_Input is the input argument to a com.atproto.server.resetPassword call. +type ServerResetPassword_Input struct { + Password string `json:"password" cborgen:"password"` + Token string `json:"token" cborgen:"token"` +} + +// ServerResetPassword calls the XRPC method "com.atproto.server.resetPassword". +func ServerResetPassword(ctx context.Context, c lexutil.LexClient, input *ServerResetPassword_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.resetPassword", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverrevokeAppPassword.go b/api/atproto/serverrevokeAppPassword.go new file mode 100644 index 000000000..3cba9c49c --- /dev/null +++ b/api/atproto/serverrevokeAppPassword.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.revokeAppPassword + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerRevokeAppPassword_Input is the input argument to a com.atproto.server.revokeAppPassword call. +type ServerRevokeAppPassword_Input struct { + Name string `json:"name" cborgen:"name"` +} + +// ServerRevokeAppPassword calls the XRPC method "com.atproto.server.revokeAppPassword". +func ServerRevokeAppPassword(ctx context.Context, c lexutil.LexClient, input *ServerRevokeAppPassword_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.revokeAppPassword", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverupdateEmail.go b/api/atproto/serverupdateEmail.go new file mode 100644 index 000000000..e79fa78eb --- /dev/null +++ b/api/atproto/serverupdateEmail.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.server.updateEmail + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerUpdateEmail_Input is the input argument to a com.atproto.server.updateEmail call. +type ServerUpdateEmail_Input struct { + Email string `json:"email" cborgen:"email"` + EmailAuthFactor *bool `json:"emailAuthFactor,omitempty" cborgen:"emailAuthFactor,omitempty"` + // token: Requires a token from com.atproto.sever.requestEmailUpdate if the account's email has been confirmed. + Token *string `json:"token,omitempty" cborgen:"token,omitempty"` +} + +// ServerUpdateEmail calls the XRPC method "com.atproto.server.updateEmail". +func ServerUpdateEmail(ctx context.Context, c lexutil.LexClient, input *ServerUpdateEmail_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.server.updateEmail", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/sessioncreate.go b/api/atproto/sessioncreate.go deleted file mode 100644 index 8e0c1fce0..000000000 --- a/api/atproto/sessioncreate.go +++ /dev/null @@ -1,35 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.session.create - -func init() { -} - -type SessionCreate_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Handle string `json:"handle" cborgen:"handle"` - Password string `json:"password" cborgen:"password"` -} - -type SessionCreate_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` - Did string `json:"did" cborgen:"did"` - Handle string `json:"handle" cborgen:"handle"` - RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` -} - -func SessionCreate(ctx context.Context, c *xrpc.Client, input *SessionCreate_Input) (*SessionCreate_Output, error) { - var out SessionCreate_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.session.create", nil, input, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/sessiondelete.go b/api/atproto/sessiondelete.go deleted file mode 100644 index 9c969ed07..000000000 --- a/api/atproto/sessiondelete.go +++ /dev/null @@ -1,19 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.session.delete - -func init() { -} -func SessionDelete(ctx context.Context, c *xrpc.Client) error { - if err := c.Do(ctx, xrpc.Procedure, "", "com.atproto.session.delete", nil, nil, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/sessionget.go b/api/atproto/sessionget.go deleted file mode 100644 index 876a9447e..000000000 --- a/api/atproto/sessionget.go +++ /dev/null @@ -1,27 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.session.get - -func init() { -} - -type SessionGet_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Did string `json:"did" cborgen:"did"` - Handle string `json:"handle" cborgen:"handle"` -} - -func SessionGet(ctx context.Context, c *xrpc.Client) (*SessionGet_Output, error) { - var out SessionGet_Output - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.session.get", nil, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/sessionrefresh.go b/api/atproto/sessionrefresh.go deleted file mode 100644 index feb8c8e49..000000000 --- a/api/atproto/sessionrefresh.go +++ /dev/null @@ -1,29 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.session.refresh - -func init() { -} - -type SessionRefresh_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` - Did string `json:"did" cborgen:"did"` - Handle string `json:"handle" cborgen:"handle"` - RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` -} - -func SessionRefresh(ctx context.Context, c *xrpc.Client) (*SessionRefresh_Output, error) { - var out SessionRefresh_Output - if err := c.Do(ctx, xrpc.Procedure, "", "com.atproto.session.refresh", nil, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/syncdefs.go b/api/atproto/syncdefs.go new file mode 100644 index 000000000..c55b26cfa --- /dev/null +++ b/api/atproto/syncdefs.go @@ -0,0 +1,5 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.defs + +package atproto diff --git a/api/atproto/syncgetBlob.go b/api/atproto/syncgetBlob.go new file mode 100644 index 000000000..0ce0cef45 --- /dev/null +++ b/api/atproto/syncgetBlob.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getBlob + +package atproto + +import ( + "bytes" + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncGetBlob calls the XRPC method "com.atproto.sync.getBlob". +// +// cid: The CID of the blob to fetch +// did: The DID of the account. +func SyncGetBlob(ctx context.Context, c lexutil.LexClient, cid string, did string) ([]byte, error) { + buf := new(bytes.Buffer) + + params := map[string]interface{}{} + params["cid"] = cid + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getBlob", params, nil, buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/api/atproto/syncgetBlocks.go b/api/atproto/syncgetBlocks.go new file mode 100644 index 000000000..3964c1a17 --- /dev/null +++ b/api/atproto/syncgetBlocks.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getBlocks + +package atproto + +import ( + "bytes" + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncGetBlocks calls the XRPC method "com.atproto.sync.getBlocks". +// +// did: The DID of the repo. +func SyncGetBlocks(ctx context.Context, c lexutil.LexClient, cids []string, did string) ([]byte, error) { + buf := new(bytes.Buffer) + + params := map[string]interface{}{} + params["cids"] = cids + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getBlocks", params, nil, buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/api/atproto/syncgetCheckout.go b/api/atproto/syncgetCheckout.go new file mode 100644 index 000000000..6c3ba0981 --- /dev/null +++ b/api/atproto/syncgetCheckout.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getCheckout + +package atproto + +import ( + "bytes" + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncGetCheckout calls the XRPC method "com.atproto.sync.getCheckout". +// +// did: The DID of the repo. +func SyncGetCheckout(ctx context.Context, c lexutil.LexClient, did string) ([]byte, error) { + buf := new(bytes.Buffer) + + params := map[string]interface{}{} + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getCheckout", params, nil, buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/api/atproto/syncgetHead.go b/api/atproto/syncgetHead.go new file mode 100644 index 000000000..efb90e36b --- /dev/null +++ b/api/atproto/syncgetHead.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getHead + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncGetHead_Output is the output of a com.atproto.sync.getHead call. +type SyncGetHead_Output struct { + Root string `json:"root" cborgen:"root"` +} + +// SyncGetHead calls the XRPC method "com.atproto.sync.getHead". +// +// did: The DID of the repo. +func SyncGetHead(ctx context.Context, c lexutil.LexClient, did string) (*SyncGetHead_Output, error) { + var out SyncGetHead_Output + + params := map[string]interface{}{} + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getHead", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/syncgetHostStatus.go b/api/atproto/syncgetHostStatus.go new file mode 100644 index 000000000..08dc4ad4c --- /dev/null +++ b/api/atproto/syncgetHostStatus.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getHostStatus + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncGetHostStatus_Output is the output of a com.atproto.sync.getHostStatus call. +type SyncGetHostStatus_Output struct { + // accountCount: Number of accounts on the server which are associated with the upstream host. Note that the upstream may actually have more accounts. + AccountCount *int64 `json:"accountCount,omitempty" cborgen:"accountCount,omitempty"` + Hostname string `json:"hostname" cborgen:"hostname"` + // seq: Recent repo stream event sequence number. May be delayed from actual stream processing (eg, persisted cursor not in-memory cursor). + Seq *int64 `json:"seq,omitempty" cborgen:"seq,omitempty"` + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +// SyncGetHostStatus calls the XRPC method "com.atproto.sync.getHostStatus". +// +// hostname: Hostname of the host (eg, PDS or relay) being queried. +func SyncGetHostStatus(ctx context.Context, c lexutil.LexClient, hostname string) (*SyncGetHostStatus_Output, error) { + var out SyncGetHostStatus_Output + + params := map[string]interface{}{} + params["hostname"] = hostname + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getHostStatus", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/syncgetLatestCommit.go b/api/atproto/syncgetLatestCommit.go new file mode 100644 index 000000000..7a27b6e8c --- /dev/null +++ b/api/atproto/syncgetLatestCommit.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getLatestCommit + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncGetLatestCommit_Output is the output of a com.atproto.sync.getLatestCommit call. +type SyncGetLatestCommit_Output struct { + Cid string `json:"cid" cborgen:"cid"` + Rev string `json:"rev" cborgen:"rev"` +} + +// SyncGetLatestCommit calls the XRPC method "com.atproto.sync.getLatestCommit". +// +// did: The DID of the repo. +func SyncGetLatestCommit(ctx context.Context, c lexutil.LexClient, did string) (*SyncGetLatestCommit_Output, error) { + var out SyncGetLatestCommit_Output + + params := map[string]interface{}{} + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getLatestCommit", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/syncgetRecord.go b/api/atproto/syncgetRecord.go new file mode 100644 index 000000000..5c037dda3 --- /dev/null +++ b/api/atproto/syncgetRecord.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getRecord + +package atproto + +import ( + "bytes" + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncGetRecord calls the XRPC method "com.atproto.sync.getRecord". +// +// did: The DID of the repo. +// rkey: Record Key +func SyncGetRecord(ctx context.Context, c lexutil.LexClient, collection string, did string, rkey string) ([]byte, error) { + buf := new(bytes.Buffer) + + params := map[string]interface{}{} + params["collection"] = collection + params["did"] = did + params["rkey"] = rkey + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getRecord", params, nil, buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/api/atproto/syncgetRepo.go b/api/atproto/syncgetRepo.go index 89750134b..02794aa39 100644 --- a/api/atproto/syncgetRepo.go +++ b/api/atproto/syncgetRepo.go @@ -1,24 +1,29 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getRepo + +package atproto import ( "bytes" "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: com.atproto.sync.getRepo - -func init() { -} -func SyncGetRepo(ctx context.Context, c *xrpc.Client, did string, from string) ([]byte, error) { +// SyncGetRepo calls the XRPC method "com.atproto.sync.getRepo". +// +// did: The DID of the repo. +// since: The revision ('rev') of the repo to create a diff from. +func SyncGetRepo(ctx context.Context, c lexutil.LexClient, did string, since string) ([]byte, error) { buf := new(bytes.Buffer) - params := map[string]interface{}{ - "did": did, - "from": from, + params := map[string]interface{}{} + params["did"] = did + if since != "" { + params["since"] = since } - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.sync.getRepo", params, nil, buf); err != nil { + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getRepo", params, nil, buf); err != nil { return nil, err } diff --git a/api/atproto/syncgetRepoStatus.go b/api/atproto/syncgetRepoStatus.go new file mode 100644 index 000000000..6799ea1dc --- /dev/null +++ b/api/atproto/syncgetRepoStatus.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.getRepoStatus + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncGetRepoStatus_Output is the output of a com.atproto.sync.getRepoStatus call. +type SyncGetRepoStatus_Output struct { + Active bool `json:"active" cborgen:"active"` + Did string `json:"did" cborgen:"did"` + // rev: Optional field, the current rev of the repo, if active=true + Rev *string `json:"rev,omitempty" cborgen:"rev,omitempty"` + // status: If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted. + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +// SyncGetRepoStatus calls the XRPC method "com.atproto.sync.getRepoStatus". +// +// did: The DID of the repo. +func SyncGetRepoStatus(ctx context.Context, c lexutil.LexClient, did string) (*SyncGetRepoStatus_Output, error) { + var out SyncGetRepoStatus_Output + + params := map[string]interface{}{} + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.getRepoStatus", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/syncgetRoot.go b/api/atproto/syncgetRoot.go deleted file mode 100644 index ae99a2299..000000000 --- a/api/atproto/syncgetRoot.go +++ /dev/null @@ -1,30 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.sync.getRoot - -func init() { -} - -type SyncGetRoot_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Root string `json:"root" cborgen:"root"` -} - -func SyncGetRoot(ctx context.Context, c *xrpc.Client, did string) (*SyncGetRoot_Output, error) { - var out SyncGetRoot_Output - - params := map[string]interface{}{ - "did": did, - } - if err := c.Do(ctx, xrpc.Query, "", "com.atproto.sync.getRoot", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/atproto/synclistBlobs.go b/api/atproto/synclistBlobs.go new file mode 100644 index 000000000..df842b557 --- /dev/null +++ b/api/atproto/synclistBlobs.go @@ -0,0 +1,42 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.listBlobs + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncListBlobs_Output is the output of a com.atproto.sync.listBlobs call. +type SyncListBlobs_Output struct { + Cids []string `json:"cids" cborgen:"cids"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// SyncListBlobs calls the XRPC method "com.atproto.sync.listBlobs". +// +// did: The DID of the repo. +// since: Optional revision of the repo to list blobs since. +func SyncListBlobs(ctx context.Context, c lexutil.LexClient, cursor string, did string, limit int64, since string) (*SyncListBlobs_Output, error) { + var out SyncListBlobs_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + params["did"] = did + if limit != 0 { + params["limit"] = limit + } + if since != "" { + params["since"] = since + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.listBlobs", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/synclistHosts.go b/api/atproto/synclistHosts.go new file mode 100644 index 000000000..d0cf8f423 --- /dev/null +++ b/api/atproto/synclistHosts.go @@ -0,0 +1,46 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.listHosts + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncListHosts_Host is a "host" in the com.atproto.sync.listHosts schema. +type SyncListHosts_Host struct { + AccountCount *int64 `json:"accountCount,omitempty" cborgen:"accountCount,omitempty"` + // hostname: hostname of server; not a URL (no scheme) + Hostname string `json:"hostname" cborgen:"hostname"` + // seq: Recent repo stream event sequence number. May be delayed from actual stream processing (eg, persisted cursor not in-memory cursor). + Seq *int64 `json:"seq,omitempty" cborgen:"seq,omitempty"` + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +// SyncListHosts_Output is the output of a com.atproto.sync.listHosts call. +type SyncListHosts_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // hosts: Sort order is not formally specified. Recommended order is by time host was first seen by the server, with oldest first. + Hosts []*SyncListHosts_Host `json:"hosts" cborgen:"hosts"` +} + +// SyncListHosts calls the XRPC method "com.atproto.sync.listHosts". +func SyncListHosts(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*SyncListHosts_Output, error) { + var out SyncListHosts_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.listHosts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/synclistRepos.go b/api/atproto/synclistRepos.go new file mode 100644 index 000000000..8bae81b83 --- /dev/null +++ b/api/atproto/synclistRepos.go @@ -0,0 +1,46 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.listRepos + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncListRepos_Output is the output of a com.atproto.sync.listRepos call. +type SyncListRepos_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Repos []*SyncListRepos_Repo `json:"repos" cborgen:"repos"` +} + +// SyncListRepos_Repo is a "repo" in the com.atproto.sync.listRepos schema. +type SyncListRepos_Repo struct { + Active *bool `json:"active,omitempty" cborgen:"active,omitempty"` + Did string `json:"did" cborgen:"did"` + // head: Current repo commit CID + Head string `json:"head" cborgen:"head"` + Rev string `json:"rev" cborgen:"rev"` + // status: If active=false, this optional field indicates a possible reason for why the account is not active. If active=false and no status is supplied, then the host makes no claim for why the repository is no longer being hosted. + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +// SyncListRepos calls the XRPC method "com.atproto.sync.listRepos". +func SyncListRepos(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*SyncListRepos_Output, error) { + var out SyncListRepos_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.listRepos", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/synclistReposByCollection.go b/api/atproto/synclistReposByCollection.go new file mode 100644 index 000000000..9df25f53c --- /dev/null +++ b/api/atproto/synclistReposByCollection.go @@ -0,0 +1,43 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.listReposByCollection + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncListReposByCollection_Output is the output of a com.atproto.sync.listReposByCollection call. +type SyncListReposByCollection_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Repos []*SyncListReposByCollection_Repo `json:"repos" cborgen:"repos"` +} + +// SyncListReposByCollection_Repo is a "repo" in the com.atproto.sync.listReposByCollection schema. +type SyncListReposByCollection_Repo struct { + Did string `json:"did" cborgen:"did"` +} + +// SyncListReposByCollection calls the XRPC method "com.atproto.sync.listReposByCollection". +// +// limit: Maximum size of response set. Recommend setting a large maximum (1000+) when enumerating large DID lists. +func SyncListReposByCollection(ctx context.Context, c lexutil.LexClient, collection string, cursor string, limit int64) (*SyncListReposByCollection_Output, error) { + var out SyncListReposByCollection_Output + + params := map[string]interface{}{} + params["collection"] = collection + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.sync.listReposByCollection", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/syncnotifyOfUpdate.go b/api/atproto/syncnotifyOfUpdate.go new file mode 100644 index 000000000..c79bf8996 --- /dev/null +++ b/api/atproto/syncnotifyOfUpdate.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.notifyOfUpdate + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncNotifyOfUpdate_Input is the input argument to a com.atproto.sync.notifyOfUpdate call. +type SyncNotifyOfUpdate_Input struct { + // hostname: Hostname of the current service (usually a PDS) that is notifying of update. + Hostname string `json:"hostname" cborgen:"hostname"` +} + +// SyncNotifyOfUpdate calls the XRPC method "com.atproto.sync.notifyOfUpdate". +func SyncNotifyOfUpdate(ctx context.Context, c lexutil.LexClient, input *SyncNotifyOfUpdate_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.sync.notifyOfUpdate", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/syncrequestCrawl.go b/api/atproto/syncrequestCrawl.go new file mode 100644 index 000000000..3e0a470b7 --- /dev/null +++ b/api/atproto/syncrequestCrawl.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.requestCrawl + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncRequestCrawl_Input is the input argument to a com.atproto.sync.requestCrawl call. +type SyncRequestCrawl_Input struct { + // hostname: Hostname of the current service (eg, PDS) that is requesting to be crawled. + Hostname string `json:"hostname" cborgen:"hostname"` +} + +// SyncRequestCrawl calls the XRPC method "com.atproto.sync.requestCrawl". +func SyncRequestCrawl(ctx context.Context, c lexutil.LexClient, input *SyncRequestCrawl_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.sync.requestCrawl", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/syncsubscribeRepos.go b/api/atproto/syncsubscribeRepos.go new file mode 100644 index 000000000..39d0c7f3d --- /dev/null +++ b/api/atproto/syncsubscribeRepos.go @@ -0,0 +1,95 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.sync.subscribeRepos + +package atproto + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SyncSubscribeRepos_Account is a "account" in the com.atproto.sync.subscribeRepos schema. +// +// Represents a change to an account's status on a host (eg, PDS or Relay). The semantics of this event are that the status is at the host which emitted the event, not necessarily that at the currently active PDS. Eg, a Relay takedown would emit a takedown with active=false, even if the PDS is still active. +type SyncSubscribeRepos_Account struct { + // active: Indicates that the account has a repository which can be fetched from the host that emitted this event. + Active bool `json:"active" cborgen:"active"` + Did string `json:"did" cborgen:"did"` + Seq int64 `json:"seq" cborgen:"seq"` + // status: If active=false, this optional field indicates a reason for why the account is not active. + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` + Time string `json:"time" cborgen:"time"` +} + +// SyncSubscribeRepos_Commit is a "commit" in the com.atproto.sync.subscribeRepos schema. +// +// Represents an update of repository state. Note that empty commits are allowed, which include no repo data changes, but an update to rev and signature. +type SyncSubscribeRepos_Commit struct { + Blobs []lexutil.LexLink `json:"blobs" cborgen:"blobs"` + // blocks: CAR file containing relevant blocks, as a diff since the previous repo state. The commit must be included as a block, and the commit block CID must be the first entry in the CAR header 'roots' list. + Blocks lexutil.LexBytes `json:"blocks,omitempty" cborgen:"blocks,omitempty"` + // commit: Repo commit object CID. + Commit lexutil.LexLink `json:"commit" cborgen:"commit"` + Ops []*SyncSubscribeRepos_RepoOp `json:"ops" cborgen:"ops"` + // prevData: The root CID of the MST tree for the previous commit from this repo (indicated by the 'since' revision field in this message). Corresponds to the 'data' field in the repo commit object. NOTE: this field is effectively required for the 'inductive' version of firehose. + PrevData *lexutil.LexLink `json:"prevData,omitempty" cborgen:"prevData,omitempty"` + // rebase: DEPRECATED -- unused + Rebase bool `json:"rebase" cborgen:"rebase"` + // repo: The repo this event comes from. Note that all other message types name this field 'did'. + Repo string `json:"repo" cborgen:"repo"` + // rev: The rev of the emitted commit. Note that this information is also in the commit object included in blocks, unless this is a tooBig event. + Rev string `json:"rev" cborgen:"rev"` + // seq: The stream sequence number of this message. + Seq int64 `json:"seq" cborgen:"seq"` + // since: The rev of the last emitted commit from this repo (if any). + Since *string `json:"since" cborgen:"since"` + // time: Timestamp of when this message was originally broadcast. + Time string `json:"time" cborgen:"time"` + // tooBig: DEPRECATED -- replaced by #sync event and data limits. Indicates that this commit contained too many ops, or data size was too large. Consumers will need to make a separate request to get missing data. + TooBig bool `json:"tooBig" cborgen:"tooBig"` +} + +// SyncSubscribeRepos_Identity is a "identity" in the com.atproto.sync.subscribeRepos schema. +// +// Represents a change to an account's identity. Could be an updated handle, signing key, or pds hosting endpoint. Serves as a prod to all downstream services to refresh their identity cache. +type SyncSubscribeRepos_Identity struct { + Did string `json:"did" cborgen:"did"` + // handle: The current handle for the account, or 'handle.invalid' if validation fails. This field is optional, might have been validated or passed-through from an upstream source. Semantics and behaviors for PDS vs Relay may evolve in the future; see atproto specs for more details. + Handle *string `json:"handle,omitempty" cborgen:"handle,omitempty"` + Seq int64 `json:"seq" cborgen:"seq"` + Time string `json:"time" cborgen:"time"` +} + +// SyncSubscribeRepos_Info is a "info" in the com.atproto.sync.subscribeRepos schema. +type SyncSubscribeRepos_Info struct { + Message *string `json:"message,omitempty" cborgen:"message,omitempty"` + Name string `json:"name" cborgen:"name"` +} + +// SyncSubscribeRepos_RepoOp is a "repoOp" in the com.atproto.sync.subscribeRepos schema. +// +// A repo operation, ie a mutation of a single record. +type SyncSubscribeRepos_RepoOp struct { + Action string `json:"action" cborgen:"action"` + // cid: For creates and updates, the new record CID. For deletions, null. + Cid *lexutil.LexLink `json:"cid" cborgen:"cid"` + Path string `json:"path" cborgen:"path"` + // prev: For updates and deletes, the previous record CID (required for inductive firehose). For creations, field should not be defined. + Prev *lexutil.LexLink `json:"prev,omitempty" cborgen:"prev,omitempty"` +} + +// SyncSubscribeRepos_Sync is a "sync" in the com.atproto.sync.subscribeRepos schema. +// +// Updates the repo to a new state, without necessarily including that state on the firehose. Used to recover from broken commit streams, data loss incidents, or in situations where upstream host does not know recent state of the repository. +type SyncSubscribeRepos_Sync struct { + // blocks: CAR file containing the commit, as a block. The CAR header must include the commit block CID as the first 'root'. + Blocks lexutil.LexBytes `json:"blocks,omitempty" cborgen:"blocks,omitempty"` + // did: The account this repo event corresponds to. Must match that in the commit object. + Did string `json:"did" cborgen:"did"` + // rev: The rev of the commit. This value must match that in the commit object. + Rev string `json:"rev" cborgen:"rev"` + // seq: The stream sequence number of this message. + Seq int64 `json:"seq" cborgen:"seq"` + // time: Timestamp of when this message was originally broadcast. + Time string `json:"time" cborgen:"time"` +} diff --git a/api/atproto/syncupdateRepo.go b/api/atproto/syncupdateRepo.go deleted file mode 100644 index ed9f935aa..000000000 --- a/api/atproto/syncupdateRepo.go +++ /dev/null @@ -1,24 +0,0 @@ -package schemagen - -import ( - "context" - "io" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: com.atproto.sync.updateRepo - -func init() { -} -func SyncUpdateRepo(ctx context.Context, c *xrpc.Client, input io.Reader, did string) error { - - params := map[string]interface{}{ - "did": did, - } - if err := c.Do(ctx, xrpc.Procedure, "application/cbor", "com.atproto.sync.updateRepo", params, input, nil); err != nil { - return err - } - - return nil -} diff --git a/api/atproto/tempaddReservedHandle.go b/api/atproto/tempaddReservedHandle.go new file mode 100644 index 000000000..a0b361018 --- /dev/null +++ b/api/atproto/tempaddReservedHandle.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.temp.addReservedHandle + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TempAddReservedHandle_Input is the input argument to a com.atproto.temp.addReservedHandle call. +type TempAddReservedHandle_Input struct { + Handle string `json:"handle" cborgen:"handle"` +} + +// TempAddReservedHandle_Output is the output of a com.atproto.temp.addReservedHandle call. +type TempAddReservedHandle_Output struct { +} + +// TempAddReservedHandle calls the XRPC method "com.atproto.temp.addReservedHandle". +func TempAddReservedHandle(ctx context.Context, c lexutil.LexClient, input *TempAddReservedHandle_Input) (*TempAddReservedHandle_Output, error) { + var out TempAddReservedHandle_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.temp.addReservedHandle", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/tempcheckHandleAvailability.go b/api/atproto/tempcheckHandleAvailability.go new file mode 100644 index 000000000..02d9597ec --- /dev/null +++ b/api/atproto/tempcheckHandleAvailability.go @@ -0,0 +1,101 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.temp.checkHandleAvailability + +package atproto + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TempCheckHandleAvailability_Output is the output of a com.atproto.temp.checkHandleAvailability call. +type TempCheckHandleAvailability_Output struct { + // handle: Echo of the input handle. + Handle string `json:"handle" cborgen:"handle"` + Result *TempCheckHandleAvailability_Output_Result `json:"result" cborgen:"result"` +} + +type TempCheckHandleAvailability_Output_Result struct { + TempCheckHandleAvailability_ResultAvailable *TempCheckHandleAvailability_ResultAvailable + TempCheckHandleAvailability_ResultUnavailable *TempCheckHandleAvailability_ResultUnavailable +} + +func (t *TempCheckHandleAvailability_Output_Result) MarshalJSON() ([]byte, error) { + if t.TempCheckHandleAvailability_ResultAvailable != nil { + t.TempCheckHandleAvailability_ResultAvailable.LexiconTypeID = "com.atproto.temp.checkHandleAvailability#resultAvailable" + return json.Marshal(t.TempCheckHandleAvailability_ResultAvailable) + } + if t.TempCheckHandleAvailability_ResultUnavailable != nil { + t.TempCheckHandleAvailability_ResultUnavailable.LexiconTypeID = "com.atproto.temp.checkHandleAvailability#resultUnavailable" + return json.Marshal(t.TempCheckHandleAvailability_ResultUnavailable) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *TempCheckHandleAvailability_Output_Result) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.temp.checkHandleAvailability#resultAvailable": + t.TempCheckHandleAvailability_ResultAvailable = new(TempCheckHandleAvailability_ResultAvailable) + return json.Unmarshal(b, t.TempCheckHandleAvailability_ResultAvailable) + case "com.atproto.temp.checkHandleAvailability#resultUnavailable": + t.TempCheckHandleAvailability_ResultUnavailable = new(TempCheckHandleAvailability_ResultUnavailable) + return json.Unmarshal(b, t.TempCheckHandleAvailability_ResultUnavailable) + default: + return nil + } +} + +// TempCheckHandleAvailability_ResultAvailable is a "resultAvailable" in the com.atproto.temp.checkHandleAvailability schema. +// +// Indicates the provided handle is available. +type TempCheckHandleAvailability_ResultAvailable struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.temp.checkHandleAvailability#resultAvailable"` +} + +// TempCheckHandleAvailability_ResultUnavailable is a "resultUnavailable" in the com.atproto.temp.checkHandleAvailability schema. +// +// Indicates the provided handle is unavailable and gives suggestions of available handles. +type TempCheckHandleAvailability_ResultUnavailable struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=com.atproto.temp.checkHandleAvailability#resultUnavailable"` + // suggestions: List of suggested handles based on the provided inputs. + Suggestions []*TempCheckHandleAvailability_Suggestion `json:"suggestions" cborgen:"suggestions"` +} + +// TempCheckHandleAvailability_Suggestion is a "suggestion" in the com.atproto.temp.checkHandleAvailability schema. +type TempCheckHandleAvailability_Suggestion struct { + Handle string `json:"handle" cborgen:"handle"` + // method: Method used to build this suggestion. Should be considered opaque to clients. Can be used for metrics. + Method string `json:"method" cborgen:"method"` +} + +// TempCheckHandleAvailability calls the XRPC method "com.atproto.temp.checkHandleAvailability". +// +// birthDate: User-provided birth date. Might be used to build handle suggestions. +// email: User-provided email. Might be used to build handle suggestions. +// handle: Tentative handle. Will be checked for availability or used to build handle suggestions. +func TempCheckHandleAvailability(ctx context.Context, c lexutil.LexClient, birthDate string, email string, handle string) (*TempCheckHandleAvailability_Output, error) { + var out TempCheckHandleAvailability_Output + + params := map[string]interface{}{} + if birthDate != "" { + params["birthDate"] = birthDate + } + if email != "" { + params["email"] = email + } + params["handle"] = handle + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.temp.checkHandleAvailability", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/tempcheckSignupQueue.go b/api/atproto/tempcheckSignupQueue.go new file mode 100644 index 000000000..1405a7b50 --- /dev/null +++ b/api/atproto/tempcheckSignupQueue.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.temp.checkSignupQueue + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TempCheckSignupQueue_Output is the output of a com.atproto.temp.checkSignupQueue call. +type TempCheckSignupQueue_Output struct { + Activated bool `json:"activated" cborgen:"activated"` + EstimatedTimeMs *int64 `json:"estimatedTimeMs,omitempty" cborgen:"estimatedTimeMs,omitempty"` + PlaceInQueue *int64 `json:"placeInQueue,omitempty" cborgen:"placeInQueue,omitempty"` +} + +// TempCheckSignupQueue calls the XRPC method "com.atproto.temp.checkSignupQueue". +func TempCheckSignupQueue(ctx context.Context, c lexutil.LexClient) (*TempCheckSignupQueue_Output, error) { + var out TempCheckSignupQueue_Output + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.temp.checkSignupQueue", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/tempdereferenceScope.go b/api/atproto/tempdereferenceScope.go new file mode 100644 index 000000000..06c32a712 --- /dev/null +++ b/api/atproto/tempdereferenceScope.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.temp.dereferenceScope + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TempDereferenceScope_Output is the output of a com.atproto.temp.dereferenceScope call. +type TempDereferenceScope_Output struct { + // scope: The full oauth permission scope + Scope string `json:"scope" cborgen:"scope"` +} + +// TempDereferenceScope calls the XRPC method "com.atproto.temp.dereferenceScope". +// +// scope: The scope reference (starts with 'ref:') +func TempDereferenceScope(ctx context.Context, c lexutil.LexClient, scope string) (*TempDereferenceScope_Output, error) { + var out TempDereferenceScope_Output + + params := map[string]interface{}{} + params["scope"] = scope + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.temp.dereferenceScope", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/tempfetchLabels.go b/api/atproto/tempfetchLabels.go new file mode 100644 index 000000000..ec62a8d36 --- /dev/null +++ b/api/atproto/tempfetchLabels.go @@ -0,0 +1,34 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.temp.fetchLabels + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TempFetchLabels_Output is the output of a com.atproto.temp.fetchLabels call. +type TempFetchLabels_Output struct { + Labels []*LabelDefs_Label `json:"labels" cborgen:"labels"` +} + +// TempFetchLabels calls the XRPC method "com.atproto.temp.fetchLabels". +func TempFetchLabels(ctx context.Context, c lexutil.LexClient, limit int64, since int64) (*TempFetchLabels_Output, error) { + var out TempFetchLabels_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if since != 0 { + params["since"] = since + } + if err := c.LexDo(ctx, lexutil.Query, "", "com.atproto.temp.fetchLabels", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/temprequestPhoneVerification.go b/api/atproto/temprequestPhoneVerification.go new file mode 100644 index 000000000..a4b27fd63 --- /dev/null +++ b/api/atproto/temprequestPhoneVerification.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.temp.requestPhoneVerification + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TempRequestPhoneVerification_Input is the input argument to a com.atproto.temp.requestPhoneVerification call. +type TempRequestPhoneVerification_Input struct { + PhoneNumber string `json:"phoneNumber" cborgen:"phoneNumber"` +} + +// TempRequestPhoneVerification calls the XRPC method "com.atproto.temp.requestPhoneVerification". +func TempRequestPhoneVerification(ctx context.Context, c lexutil.LexClient, input *TempRequestPhoneVerification_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.temp.requestPhoneVerification", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/temprevokeAccountCredentials.go b/api/atproto/temprevokeAccountCredentials.go new file mode 100644 index 000000000..6bfdf58b4 --- /dev/null +++ b/api/atproto/temprevokeAccountCredentials.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: com.atproto.temp.revokeAccountCredentials + +package atproto + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TempRevokeAccountCredentials_Input is the input argument to a com.atproto.temp.revokeAccountCredentials call. +type TempRevokeAccountCredentials_Input struct { + Account string `json:"account" cborgen:"account"` +} + +// TempRevokeAccountCredentials calls the XRPC method "com.atproto.temp.revokeAccountCredentials". +func TempRevokeAccountCredentials(ctx context.Context, c lexutil.LexClient, input *TempRevokeAccountCredentials_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "com.atproto.temp.revokeAccountCredentials", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky.go b/api/bsky.go deleted file mode 100644 index 391a9a361..000000000 --- a/api/bsky.go +++ /dev/null @@ -1,186 +0,0 @@ -package api - -import ( - "context" - "fmt" - - bsky "github.com/bluesky-social/indigo/api/bsky" - "github.com/bluesky-social/indigo/xrpc" -) - -type BskyApp struct { - C *xrpc.Client -} - -type PostEntity struct { - Index *TextSlice `json:"index" cborgen:"index"` - Type string `json:"type" cborgen:"type"` - Value string `json:"value" cborgen:"value"` -} - -type TextSlice struct { - Start int64 `json:"start" cborgen:"start"` - End int64 `json:"end" cborgen:"end"` -} - -type ReplyRef struct { - Root PostRef `json:"root" cborgen:"root"` - Parent PostRef `json:"parent" cborgen:"parent"` -} - -type PostRecord struct { - Type string `json:"$type,omitempty" cborgen:"$type"` - Text string `json:"text" cborgen:"text"` - Entities []*PostEntity `json:"entities,omitempty" cborgen:"entities"` - Reply *ReplyRef `json:"reply,omitempty" cborgen:"reply"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` -} - -type PostRef struct { - Uri string `json:"uri"` - Cid string `json:"cid"` -} - -type GetTimelineResp struct { - Cursor string `json:"cursor"` - Feed []FeedItem `json:"feed"` -} - -type FeedItem struct { - Uri string `json:"uri"` - Cid string `json:"cid"` - Author *User `json:"author"` - RepostedBy *User `json:"repostedBy"` - MyState *MyState `json:"myState"` - Record interface{} `json:"record"` -} - -type MyState struct { - Repost string `json:"repost"` - Upvote string `json:"upvote"` - Downvote string `json:"downvote"` -} - -type Declaration struct { - Cid string `json:"cid"` - ActorType string `json:"actorType"` -} - -type User struct { - Did string `json:"did"` - Handle string `json:"handle"` - DisplayName string `json:"displayName"` - Declaration *Declaration `json:"declaration"` -} - -func (b *BskyApp) FeedGetTimeline(ctx context.Context, algo string, limit int, before *string) (*bsky.FeedGetTimeline_Output, error) { - params := map[string]interface{}{ - "algorithm": algo, - "limit": limit, - } - - if before != nil { - params["before"] = *before - } - - var out bsky.FeedGetTimeline_Output - if err := b.C.Do(ctx, xrpc.Query, encJson, "app.bsky.feed.getTimeline", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} - -func (b *BskyApp) FeedGetAuthorFeed(ctx context.Context, author string, limit int, before *string) (*bsky.FeedGetAuthorFeed_Output, error) { - params := map[string]interface{}{ - "author": author, - "limit": limit, - } - - if before != nil { - params["before"] = *before - } - - var out bsky.FeedGetAuthorFeed_Output - if err := b.C.Do(ctx, xrpc.Query, encJson, "app.bsky.feed.getAuthorFeed", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} - -type GSADeclaration struct { - Cid string `json:"cid"` - ActorType string `json:"actorType"` -} - -type GetSuggestionsActor struct { - Did string `json:"did"` - Declaration *GSADeclaration `json:"declaration"` - Handle string `json:"handle"` - DisplayName string `json:"displayName"` - Description string `json:"description"` - IndexedAt string `json:"indexedAt"` -} - -type GetSuggestionsResp struct { - Cursor string `json:"cursor"` - Actors []GetSuggestionsActor `json:"actors"` -} - -func (b *BskyApp) ActorGetSuggestions(ctx context.Context, limit int, cursor *string) (*GetSuggestionsResp, error) { - params := map[string]interface{}{ - "limit": limit, - } - - if cursor != nil { - params["cursor"] = *cursor - } - - var out GetSuggestionsResp - if err := b.C.Do(ctx, xrpc.Query, "", "app.bsky.actor.getSuggestions", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} - -func (b *BskyApp) FeedSetVote(ctx context.Context, subject *PostRef, direction string) error { - body := map[string]interface{}{ - "subject": subject, - "direction": direction, - } - - var out map[string]interface{} - if err := b.C.Do(ctx, xrpc.Procedure, encJson, "app.bsky.feed.setVote", nil, body, &out); err != nil { - return err - } - - fmt.Println(out) - - return nil -} - -type GetFollowsResp struct { - Subject *User `json:"subject"` - Cursor string `json:"cursor"` - Follows []User `json:"follows"` -} - -func (b *BskyApp) GraphGetFollows(ctx context.Context, user string, limit int, before *string) (*GetFollowsResp, error) { - params := map[string]interface{}{ - "user": user, - "limit": limit, - } - - if before != nil { - params["before"] = *before - } - - var out GetFollowsResp - if err := b.C.Do(ctx, xrpc.Query, "", "app.bsky.graph.getFollows", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/actorcreateScene.go b/api/bsky/actorcreateScene.go deleted file mode 100644 index 9b242be63..000000000 --- a/api/bsky/actorcreateScene.go +++ /dev/null @@ -1,34 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.actor.createScene - -func init() { -} - -type ActorCreateScene_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Handle string `json:"handle" cborgen:"handle"` - RecoveryKey *string `json:"recoveryKey,omitempty" cborgen:"recoveryKey"` -} - -type ActorCreateScene_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - Handle string `json:"handle" cborgen:"handle"` -} - -func ActorCreateScene(ctx context.Context, c *xrpc.Client, input *ActorCreateScene_Input) (*ActorCreateScene_Output, error) { - var out ActorCreateScene_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "app.bsky.actor.createScene", nil, input, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/actordefs.go b/api/bsky/actordefs.go new file mode 100644 index 000000000..51b9881bf --- /dev/null +++ b/api/bsky/actordefs.go @@ -0,0 +1,562 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.defs + +package bsky + +import ( + "encoding/json" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ActorDefs_AdultContentPref is a "adultContentPref" in the app.bsky.actor.defs schema. +type ActorDefs_AdultContentPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#adultContentPref"` + Enabled bool `json:"enabled" cborgen:"enabled"` +} + +// ActorDefs_BskyAppProgressGuide is a "bskyAppProgressGuide" in the app.bsky.actor.defs schema. +// +// If set, an active progress guide. Once completed, can be set to undefined. Should have unspecced fields tracking progress. +type ActorDefs_BskyAppProgressGuide struct { + Guide string `json:"guide" cborgen:"guide"` +} + +// ActorDefs_BskyAppStatePref is a "bskyAppStatePref" in the app.bsky.actor.defs schema. +// +// A grab bag of state that's specific to the bsky.app program. Third-party apps shouldn't use this. +type ActorDefs_BskyAppStatePref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#bskyAppStatePref"` + ActiveProgressGuide *ActorDefs_BskyAppProgressGuide `json:"activeProgressGuide,omitempty" cborgen:"activeProgressGuide,omitempty"` + // nuxs: Storage for NUXs the user has encountered. + Nuxs []*ActorDefs_Nux `json:"nuxs,omitempty" cborgen:"nuxs,omitempty"` + // queuedNudges: An array of tokens which identify nudges (modals, popups, tours, highlight dots) that should be shown to the user. + QueuedNudges []string `json:"queuedNudges,omitempty" cborgen:"queuedNudges,omitempty"` +} + +// ActorDefs_ContentLabelPref is a "contentLabelPref" in the app.bsky.actor.defs schema. +type ActorDefs_ContentLabelPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#contentLabelPref"` + Label string `json:"label" cborgen:"label"` + // labelerDid: Which labeler does this preference apply to? If undefined, applies globally. + LabelerDid *string `json:"labelerDid,omitempty" cborgen:"labelerDid,omitempty"` + Visibility string `json:"visibility" cborgen:"visibility"` +} + +// ActorDefs_FeedViewPref is a "feedViewPref" in the app.bsky.actor.defs schema. +type ActorDefs_FeedViewPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#feedViewPref"` + // feed: The URI of the feed, or an identifier which describes the feed. + Feed string `json:"feed" cborgen:"feed"` + // hideQuotePosts: Hide quote posts in the feed. + HideQuotePosts *bool `json:"hideQuotePosts,omitempty" cborgen:"hideQuotePosts,omitempty"` + // hideReplies: Hide replies in the feed. + HideReplies *bool `json:"hideReplies,omitempty" cborgen:"hideReplies,omitempty"` + // hideRepliesByLikeCount: Hide replies in the feed if they do not have this number of likes. + HideRepliesByLikeCount *int64 `json:"hideRepliesByLikeCount,omitempty" cborgen:"hideRepliesByLikeCount,omitempty"` + // hideRepliesByUnfollowed: Hide replies in the feed if they are not by followed users. + HideRepliesByUnfollowed *bool `json:"hideRepliesByUnfollowed,omitempty" cborgen:"hideRepliesByUnfollowed,omitempty"` + // hideReposts: Hide reposts in the feed. + HideReposts *bool `json:"hideReposts,omitempty" cborgen:"hideReposts,omitempty"` +} + +// ActorDefs_HiddenPostsPref is a "hiddenPostsPref" in the app.bsky.actor.defs schema. +type ActorDefs_HiddenPostsPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#hiddenPostsPref"` + // items: A list of URIs of posts the account owner has hidden. + Items []string `json:"items" cborgen:"items"` +} + +// ActorDefs_InterestsPref is a "interestsPref" in the app.bsky.actor.defs schema. +type ActorDefs_InterestsPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#interestsPref"` + // tags: A list of tags which describe the account owner's interests gathered during onboarding. + Tags []string `json:"tags" cborgen:"tags"` +} + +// ActorDefs_KnownFollowers is a "knownFollowers" in the app.bsky.actor.defs schema. +// +// The subject's followers whom you also follow +type ActorDefs_KnownFollowers struct { + Count int64 `json:"count" cborgen:"count"` + Followers []*ActorDefs_ProfileViewBasic `json:"followers" cborgen:"followers"` +} + +// ActorDefs_LabelerPrefItem is a "labelerPrefItem" in the app.bsky.actor.defs schema. +type ActorDefs_LabelerPrefItem struct { + Did string `json:"did" cborgen:"did"` +} + +// ActorDefs_LabelersPref is a "labelersPref" in the app.bsky.actor.defs schema. +type ActorDefs_LabelersPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#labelersPref"` + Labelers []*ActorDefs_LabelerPrefItem `json:"labelers" cborgen:"labelers"` +} + +// ActorDefs_MutedWord is a "mutedWord" in the app.bsky.actor.defs schema. +// +// A word that the account owner has muted. +type ActorDefs_MutedWord struct { + // actorTarget: Groups of users to apply the muted word to. If undefined, applies to all users. + ActorTarget *string `json:"actorTarget,omitempty" cborgen:"actorTarget,omitempty"` + // expiresAt: The date and time at which the muted word will expire and no longer be applied. + ExpiresAt *string `json:"expiresAt,omitempty" cborgen:"expiresAt,omitempty"` + Id *string `json:"id,omitempty" cborgen:"id,omitempty"` + // targets: The intended targets of the muted word. + Targets []*string `json:"targets" cborgen:"targets"` + // value: The muted word itself. + Value string `json:"value" cborgen:"value"` +} + +// ActorDefs_MutedWordsPref is a "mutedWordsPref" in the app.bsky.actor.defs schema. +type ActorDefs_MutedWordsPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#mutedWordsPref"` + // items: A list of words the account owner has muted. + Items []*ActorDefs_MutedWord `json:"items" cborgen:"items"` +} + +// ActorDefs_Nux is a "nux" in the app.bsky.actor.defs schema. +// +// A new user experiences (NUX) storage object +type ActorDefs_Nux struct { + Completed bool `json:"completed" cborgen:"completed"` + // data: Arbitrary data for the NUX. The structure is defined by the NUX itself. Limited to 300 characters. + Data *string `json:"data,omitempty" cborgen:"data,omitempty"` + // expiresAt: The date and time at which the NUX will expire and should be considered completed. + ExpiresAt *string `json:"expiresAt,omitempty" cborgen:"expiresAt,omitempty"` + Id string `json:"id" cborgen:"id"` +} + +// ActorDefs_PersonalDetailsPref is a "personalDetailsPref" in the app.bsky.actor.defs schema. +type ActorDefs_PersonalDetailsPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#personalDetailsPref"` + // birthDate: The birth date of account owner. + BirthDate *string `json:"birthDate,omitempty" cborgen:"birthDate,omitempty"` +} + +// ActorDefs_PostInteractionSettingsPref is a "postInteractionSettingsPref" in the app.bsky.actor.defs schema. +// +// Default post interaction settings for the account. These values should be applied as default values when creating new posts. These refs should mirror the threadgate and postgate records exactly. +type ActorDefs_PostInteractionSettingsPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#postInteractionSettingsPref"` + // postgateEmbeddingRules: Matches postgate record. List of rules defining who can embed this users posts. If value is an empty array or is undefined, no particular rules apply and anyone can embed. + PostgateEmbeddingRules []*ActorDefs_PostInteractionSettingsPref_PostgateEmbeddingRules_Elem `json:"postgateEmbeddingRules,omitempty" cborgen:"postgateEmbeddingRules,omitempty"` + // threadgateAllowRules: Matches threadgate record. List of rules defining who can reply to this users posts. If value is an empty array, no one can reply. If value is undefined, anyone can reply. + ThreadgateAllowRules []*ActorDefs_PostInteractionSettingsPref_ThreadgateAllowRules_Elem `json:"threadgateAllowRules,omitempty" cborgen:"threadgateAllowRules,omitempty"` +} + +type ActorDefs_PostInteractionSettingsPref_PostgateEmbeddingRules_Elem struct { + FeedPostgate_DisableRule *FeedPostgate_DisableRule +} + +func (t *ActorDefs_PostInteractionSettingsPref_PostgateEmbeddingRules_Elem) MarshalJSON() ([]byte, error) { + if t.FeedPostgate_DisableRule != nil { + t.FeedPostgate_DisableRule.LexiconTypeID = "app.bsky.feed.postgate#disableRule" + return json.Marshal(t.FeedPostgate_DisableRule) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ActorDefs_PostInteractionSettingsPref_PostgateEmbeddingRules_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.postgate#disableRule": + t.FeedPostgate_DisableRule = new(FeedPostgate_DisableRule) + return json.Unmarshal(b, t.FeedPostgate_DisableRule) + default: + return nil + } +} + +type ActorDefs_PostInteractionSettingsPref_ThreadgateAllowRules_Elem struct { + FeedThreadgate_MentionRule *FeedThreadgate_MentionRule + FeedThreadgate_FollowerRule *FeedThreadgate_FollowerRule + FeedThreadgate_FollowingRule *FeedThreadgate_FollowingRule + FeedThreadgate_ListRule *FeedThreadgate_ListRule +} + +func (t *ActorDefs_PostInteractionSettingsPref_ThreadgateAllowRules_Elem) MarshalJSON() ([]byte, error) { + if t.FeedThreadgate_MentionRule != nil { + t.FeedThreadgate_MentionRule.LexiconTypeID = "app.bsky.feed.threadgate#mentionRule" + return json.Marshal(t.FeedThreadgate_MentionRule) + } + if t.FeedThreadgate_FollowerRule != nil { + t.FeedThreadgate_FollowerRule.LexiconTypeID = "app.bsky.feed.threadgate#followerRule" + return json.Marshal(t.FeedThreadgate_FollowerRule) + } + if t.FeedThreadgate_FollowingRule != nil { + t.FeedThreadgate_FollowingRule.LexiconTypeID = "app.bsky.feed.threadgate#followingRule" + return json.Marshal(t.FeedThreadgate_FollowingRule) + } + if t.FeedThreadgate_ListRule != nil { + t.FeedThreadgate_ListRule.LexiconTypeID = "app.bsky.feed.threadgate#listRule" + return json.Marshal(t.FeedThreadgate_ListRule) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ActorDefs_PostInteractionSettingsPref_ThreadgateAllowRules_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.threadgate#mentionRule": + t.FeedThreadgate_MentionRule = new(FeedThreadgate_MentionRule) + return json.Unmarshal(b, t.FeedThreadgate_MentionRule) + case "app.bsky.feed.threadgate#followerRule": + t.FeedThreadgate_FollowerRule = new(FeedThreadgate_FollowerRule) + return json.Unmarshal(b, t.FeedThreadgate_FollowerRule) + case "app.bsky.feed.threadgate#followingRule": + t.FeedThreadgate_FollowingRule = new(FeedThreadgate_FollowingRule) + return json.Unmarshal(b, t.FeedThreadgate_FollowingRule) + case "app.bsky.feed.threadgate#listRule": + t.FeedThreadgate_ListRule = new(FeedThreadgate_ListRule) + return json.Unmarshal(b, t.FeedThreadgate_ListRule) + default: + return nil + } +} + +type ActorDefs_Preferences_Elem struct { + ActorDefs_AdultContentPref *ActorDefs_AdultContentPref + ActorDefs_ContentLabelPref *ActorDefs_ContentLabelPref + ActorDefs_SavedFeedsPref *ActorDefs_SavedFeedsPref + ActorDefs_SavedFeedsPrefV2 *ActorDefs_SavedFeedsPrefV2 + ActorDefs_PersonalDetailsPref *ActorDefs_PersonalDetailsPref + ActorDefs_FeedViewPref *ActorDefs_FeedViewPref + ActorDefs_ThreadViewPref *ActorDefs_ThreadViewPref + ActorDefs_InterestsPref *ActorDefs_InterestsPref + ActorDefs_MutedWordsPref *ActorDefs_MutedWordsPref + ActorDefs_HiddenPostsPref *ActorDefs_HiddenPostsPref + ActorDefs_BskyAppStatePref *ActorDefs_BskyAppStatePref + ActorDefs_LabelersPref *ActorDefs_LabelersPref + ActorDefs_PostInteractionSettingsPref *ActorDefs_PostInteractionSettingsPref + ActorDefs_VerificationPrefs *ActorDefs_VerificationPrefs +} + +func (t *ActorDefs_Preferences_Elem) MarshalJSON() ([]byte, error) { + if t.ActorDefs_AdultContentPref != nil { + t.ActorDefs_AdultContentPref.LexiconTypeID = "app.bsky.actor.defs#adultContentPref" + return json.Marshal(t.ActorDefs_AdultContentPref) + } + if t.ActorDefs_ContentLabelPref != nil { + t.ActorDefs_ContentLabelPref.LexiconTypeID = "app.bsky.actor.defs#contentLabelPref" + return json.Marshal(t.ActorDefs_ContentLabelPref) + } + if t.ActorDefs_SavedFeedsPref != nil { + t.ActorDefs_SavedFeedsPref.LexiconTypeID = "app.bsky.actor.defs#savedFeedsPref" + return json.Marshal(t.ActorDefs_SavedFeedsPref) + } + if t.ActorDefs_SavedFeedsPrefV2 != nil { + t.ActorDefs_SavedFeedsPrefV2.LexiconTypeID = "app.bsky.actor.defs#savedFeedsPrefV2" + return json.Marshal(t.ActorDefs_SavedFeedsPrefV2) + } + if t.ActorDefs_PersonalDetailsPref != nil { + t.ActorDefs_PersonalDetailsPref.LexiconTypeID = "app.bsky.actor.defs#personalDetailsPref" + return json.Marshal(t.ActorDefs_PersonalDetailsPref) + } + if t.ActorDefs_FeedViewPref != nil { + t.ActorDefs_FeedViewPref.LexiconTypeID = "app.bsky.actor.defs#feedViewPref" + return json.Marshal(t.ActorDefs_FeedViewPref) + } + if t.ActorDefs_ThreadViewPref != nil { + t.ActorDefs_ThreadViewPref.LexiconTypeID = "app.bsky.actor.defs#threadViewPref" + return json.Marshal(t.ActorDefs_ThreadViewPref) + } + if t.ActorDefs_InterestsPref != nil { + t.ActorDefs_InterestsPref.LexiconTypeID = "app.bsky.actor.defs#interestsPref" + return json.Marshal(t.ActorDefs_InterestsPref) + } + if t.ActorDefs_MutedWordsPref != nil { + t.ActorDefs_MutedWordsPref.LexiconTypeID = "app.bsky.actor.defs#mutedWordsPref" + return json.Marshal(t.ActorDefs_MutedWordsPref) + } + if t.ActorDefs_HiddenPostsPref != nil { + t.ActorDefs_HiddenPostsPref.LexiconTypeID = "app.bsky.actor.defs#hiddenPostsPref" + return json.Marshal(t.ActorDefs_HiddenPostsPref) + } + if t.ActorDefs_BskyAppStatePref != nil { + t.ActorDefs_BskyAppStatePref.LexiconTypeID = "app.bsky.actor.defs#bskyAppStatePref" + return json.Marshal(t.ActorDefs_BskyAppStatePref) + } + if t.ActorDefs_LabelersPref != nil { + t.ActorDefs_LabelersPref.LexiconTypeID = "app.bsky.actor.defs#labelersPref" + return json.Marshal(t.ActorDefs_LabelersPref) + } + if t.ActorDefs_PostInteractionSettingsPref != nil { + t.ActorDefs_PostInteractionSettingsPref.LexiconTypeID = "app.bsky.actor.defs#postInteractionSettingsPref" + return json.Marshal(t.ActorDefs_PostInteractionSettingsPref) + } + if t.ActorDefs_VerificationPrefs != nil { + t.ActorDefs_VerificationPrefs.LexiconTypeID = "app.bsky.actor.defs#verificationPrefs" + return json.Marshal(t.ActorDefs_VerificationPrefs) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ActorDefs_Preferences_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.actor.defs#adultContentPref": + t.ActorDefs_AdultContentPref = new(ActorDefs_AdultContentPref) + return json.Unmarshal(b, t.ActorDefs_AdultContentPref) + case "app.bsky.actor.defs#contentLabelPref": + t.ActorDefs_ContentLabelPref = new(ActorDefs_ContentLabelPref) + return json.Unmarshal(b, t.ActorDefs_ContentLabelPref) + case "app.bsky.actor.defs#savedFeedsPref": + t.ActorDefs_SavedFeedsPref = new(ActorDefs_SavedFeedsPref) + return json.Unmarshal(b, t.ActorDefs_SavedFeedsPref) + case "app.bsky.actor.defs#savedFeedsPrefV2": + t.ActorDefs_SavedFeedsPrefV2 = new(ActorDefs_SavedFeedsPrefV2) + return json.Unmarshal(b, t.ActorDefs_SavedFeedsPrefV2) + case "app.bsky.actor.defs#personalDetailsPref": + t.ActorDefs_PersonalDetailsPref = new(ActorDefs_PersonalDetailsPref) + return json.Unmarshal(b, t.ActorDefs_PersonalDetailsPref) + case "app.bsky.actor.defs#feedViewPref": + t.ActorDefs_FeedViewPref = new(ActorDefs_FeedViewPref) + return json.Unmarshal(b, t.ActorDefs_FeedViewPref) + case "app.bsky.actor.defs#threadViewPref": + t.ActorDefs_ThreadViewPref = new(ActorDefs_ThreadViewPref) + return json.Unmarshal(b, t.ActorDefs_ThreadViewPref) + case "app.bsky.actor.defs#interestsPref": + t.ActorDefs_InterestsPref = new(ActorDefs_InterestsPref) + return json.Unmarshal(b, t.ActorDefs_InterestsPref) + case "app.bsky.actor.defs#mutedWordsPref": + t.ActorDefs_MutedWordsPref = new(ActorDefs_MutedWordsPref) + return json.Unmarshal(b, t.ActorDefs_MutedWordsPref) + case "app.bsky.actor.defs#hiddenPostsPref": + t.ActorDefs_HiddenPostsPref = new(ActorDefs_HiddenPostsPref) + return json.Unmarshal(b, t.ActorDefs_HiddenPostsPref) + case "app.bsky.actor.defs#bskyAppStatePref": + t.ActorDefs_BskyAppStatePref = new(ActorDefs_BskyAppStatePref) + return json.Unmarshal(b, t.ActorDefs_BskyAppStatePref) + case "app.bsky.actor.defs#labelersPref": + t.ActorDefs_LabelersPref = new(ActorDefs_LabelersPref) + return json.Unmarshal(b, t.ActorDefs_LabelersPref) + case "app.bsky.actor.defs#postInteractionSettingsPref": + t.ActorDefs_PostInteractionSettingsPref = new(ActorDefs_PostInteractionSettingsPref) + return json.Unmarshal(b, t.ActorDefs_PostInteractionSettingsPref) + case "app.bsky.actor.defs#verificationPrefs": + t.ActorDefs_VerificationPrefs = new(ActorDefs_VerificationPrefs) + return json.Unmarshal(b, t.ActorDefs_VerificationPrefs) + default: + return nil + } +} + +// ActorDefs_ProfileAssociated is a "profileAssociated" in the app.bsky.actor.defs schema. +type ActorDefs_ProfileAssociated struct { + ActivitySubscription *ActorDefs_ProfileAssociatedActivitySubscription `json:"activitySubscription,omitempty" cborgen:"activitySubscription,omitempty"` + Chat *ActorDefs_ProfileAssociatedChat `json:"chat,omitempty" cborgen:"chat,omitempty"` + Feedgens *int64 `json:"feedgens,omitempty" cborgen:"feedgens,omitempty"` + Labeler *bool `json:"labeler,omitempty" cborgen:"labeler,omitempty"` + Lists *int64 `json:"lists,omitempty" cborgen:"lists,omitempty"` + StarterPacks *int64 `json:"starterPacks,omitempty" cborgen:"starterPacks,omitempty"` +} + +// ActorDefs_ProfileAssociatedActivitySubscription is a "profileAssociatedActivitySubscription" in the app.bsky.actor.defs schema. +type ActorDefs_ProfileAssociatedActivitySubscription struct { + AllowSubscriptions string `json:"allowSubscriptions" cborgen:"allowSubscriptions"` +} + +// ActorDefs_ProfileAssociatedChat is a "profileAssociatedChat" in the app.bsky.actor.defs schema. +type ActorDefs_ProfileAssociatedChat struct { + AllowIncoming string `json:"allowIncoming" cborgen:"allowIncoming"` +} + +// ActorDefs_ProfileView is a "profileView" in the app.bsky.actor.defs schema. +type ActorDefs_ProfileView struct { + Associated *ActorDefs_ProfileAssociated `json:"associated,omitempty" cborgen:"associated,omitempty"` + Avatar *string `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + // debug: Debug information for internal development + Debug *interface{} `json:"debug,omitempty" cborgen:"debug,omitempty"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + Did string `json:"did" cborgen:"did"` + DisplayName *string `json:"displayName,omitempty" cborgen:"displayName,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + IndexedAt *string `json:"indexedAt,omitempty" cborgen:"indexedAt,omitempty"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + Pronouns *string `json:"pronouns,omitempty" cborgen:"pronouns,omitempty"` + Status *ActorDefs_StatusView `json:"status,omitempty" cborgen:"status,omitempty"` + Verification *ActorDefs_VerificationState `json:"verification,omitempty" cborgen:"verification,omitempty"` + Viewer *ActorDefs_ViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// ActorDefs_ProfileViewBasic is a "profileViewBasic" in the app.bsky.actor.defs schema. +type ActorDefs_ProfileViewBasic struct { + Associated *ActorDefs_ProfileAssociated `json:"associated,omitempty" cborgen:"associated,omitempty"` + Avatar *string `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + // debug: Debug information for internal development + Debug *interface{} `json:"debug,omitempty" cborgen:"debug,omitempty"` + Did string `json:"did" cborgen:"did"` + DisplayName *string `json:"displayName,omitempty" cborgen:"displayName,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + Pronouns *string `json:"pronouns,omitempty" cborgen:"pronouns,omitempty"` + Status *ActorDefs_StatusView `json:"status,omitempty" cborgen:"status,omitempty"` + Verification *ActorDefs_VerificationState `json:"verification,omitempty" cborgen:"verification,omitempty"` + Viewer *ActorDefs_ViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// ActorDefs_ProfileViewDetailed is a "profileViewDetailed" in the app.bsky.actor.defs schema. +type ActorDefs_ProfileViewDetailed struct { + Associated *ActorDefs_ProfileAssociated `json:"associated,omitempty" cborgen:"associated,omitempty"` + Avatar *string `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + Banner *string `json:"banner,omitempty" cborgen:"banner,omitempty"` + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + // debug: Debug information for internal development + Debug *interface{} `json:"debug,omitempty" cborgen:"debug,omitempty"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + Did string `json:"did" cborgen:"did"` + DisplayName *string `json:"displayName,omitempty" cborgen:"displayName,omitempty"` + FollowersCount *int64 `json:"followersCount,omitempty" cborgen:"followersCount,omitempty"` + FollowsCount *int64 `json:"followsCount,omitempty" cborgen:"followsCount,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + IndexedAt *string `json:"indexedAt,omitempty" cborgen:"indexedAt,omitempty"` + JoinedViaStarterPack *GraphDefs_StarterPackViewBasic `json:"joinedViaStarterPack,omitempty" cborgen:"joinedViaStarterPack,omitempty"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + PinnedPost *comatproto.RepoStrongRef `json:"pinnedPost,omitempty" cborgen:"pinnedPost,omitempty"` + PostsCount *int64 `json:"postsCount,omitempty" cborgen:"postsCount,omitempty"` + Pronouns *string `json:"pronouns,omitempty" cborgen:"pronouns,omitempty"` + Status *ActorDefs_StatusView `json:"status,omitempty" cborgen:"status,omitempty"` + Verification *ActorDefs_VerificationState `json:"verification,omitempty" cborgen:"verification,omitempty"` + Viewer *ActorDefs_ViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` + Website *string `json:"website,omitempty" cborgen:"website,omitempty"` +} + +// ActorDefs_SavedFeed is a "savedFeed" in the app.bsky.actor.defs schema. +type ActorDefs_SavedFeed struct { + Id string `json:"id" cborgen:"id"` + Pinned bool `json:"pinned" cborgen:"pinned"` + Type string `json:"type" cborgen:"type"` + Value string `json:"value" cborgen:"value"` +} + +// ActorDefs_SavedFeedsPref is a "savedFeedsPref" in the app.bsky.actor.defs schema. +type ActorDefs_SavedFeedsPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#savedFeedsPref"` + Pinned []string `json:"pinned" cborgen:"pinned"` + Saved []string `json:"saved" cborgen:"saved"` + TimelineIndex *int64 `json:"timelineIndex,omitempty" cborgen:"timelineIndex,omitempty"` +} + +// ActorDefs_SavedFeedsPrefV2 is a "savedFeedsPrefV2" in the app.bsky.actor.defs schema. +type ActorDefs_SavedFeedsPrefV2 struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#savedFeedsPrefV2"` + Items []*ActorDefs_SavedFeed `json:"items" cborgen:"items"` +} + +// ActorDefs_StatusView is a "statusView" in the app.bsky.actor.defs schema. +type ActorDefs_StatusView struct { + // embed: An optional embed associated with the status. + Embed *ActorDefs_StatusView_Embed `json:"embed,omitempty" cborgen:"embed,omitempty"` + // expiresAt: The date when this status will expire. The application might choose to no longer return the status after expiration. + ExpiresAt *string `json:"expiresAt,omitempty" cborgen:"expiresAt,omitempty"` + // isActive: True if the status is not expired, false if it is expired. Only present if expiration was set. + IsActive *bool `json:"isActive,omitempty" cborgen:"isActive,omitempty"` + Record *lexutil.LexiconTypeDecoder `json:"record" cborgen:"record"` + // status: The status for the account. + Status string `json:"status" cborgen:"status"` +} + +// An optional embed associated with the status. +type ActorDefs_StatusView_Embed struct { + EmbedExternal_View *EmbedExternal_View +} + +func (t *ActorDefs_StatusView_Embed) MarshalJSON() ([]byte, error) { + if t.EmbedExternal_View != nil { + t.EmbedExternal_View.LexiconTypeID = "app.bsky.embed.external#view" + return json.Marshal(t.EmbedExternal_View) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ActorDefs_StatusView_Embed) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.external#view": + t.EmbedExternal_View = new(EmbedExternal_View) + return json.Unmarshal(b, t.EmbedExternal_View) + default: + return nil + } +} + +// ActorDefs_ThreadViewPref is a "threadViewPref" in the app.bsky.actor.defs schema. +type ActorDefs_ThreadViewPref struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#threadViewPref"` + // sort: Sorting mode for threads. + Sort *string `json:"sort,omitempty" cborgen:"sort,omitempty"` +} + +// ActorDefs_VerificationPrefs is a "verificationPrefs" in the app.bsky.actor.defs schema. +// +// Preferences for how verified accounts appear in the app. +type ActorDefs_VerificationPrefs struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.defs#verificationPrefs"` + // hideBadges: Hide the blue check badges for verified accounts and trusted verifiers. + HideBadges *bool `json:"hideBadges,omitempty" cborgen:"hideBadges,omitempty"` +} + +// ActorDefs_VerificationState is a "verificationState" in the app.bsky.actor.defs schema. +// +// Represents the verification information about the user this object is attached to. +type ActorDefs_VerificationState struct { + // trustedVerifierStatus: The user's status as a trusted verifier. + TrustedVerifierStatus string `json:"trustedVerifierStatus" cborgen:"trustedVerifierStatus"` + // verifications: All verifications issued by trusted verifiers on behalf of this user. Verifications by untrusted verifiers are not included. + Verifications []*ActorDefs_VerificationView `json:"verifications" cborgen:"verifications"` + // verifiedStatus: The user's status as a verified account. + VerifiedStatus string `json:"verifiedStatus" cborgen:"verifiedStatus"` +} + +// ActorDefs_VerificationView is a "verificationView" in the app.bsky.actor.defs schema. +// +// An individual verification for an associated subject. +type ActorDefs_VerificationView struct { + // createdAt: Timestamp when the verification was created. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // isValid: True if the verification passes validation, otherwise false. + IsValid bool `json:"isValid" cborgen:"isValid"` + // issuer: The user who issued this verification. + Issuer string `json:"issuer" cborgen:"issuer"` + // uri: The AT-URI of the verification record. + Uri string `json:"uri" cborgen:"uri"` +} + +// ActorDefs_ViewerState is a "viewerState" in the app.bsky.actor.defs schema. +// +// Metadata about the requesting account's relationship with the subject account. Only has meaningful content for authed requests. +type ActorDefs_ViewerState struct { + // activitySubscription: This property is present only in selected cases, as an optimization. + ActivitySubscription *NotificationDefs_ActivitySubscription `json:"activitySubscription,omitempty" cborgen:"activitySubscription,omitempty"` + BlockedBy *bool `json:"blockedBy,omitempty" cborgen:"blockedBy,omitempty"` + Blocking *string `json:"blocking,omitempty" cborgen:"blocking,omitempty"` + BlockingByList *GraphDefs_ListViewBasic `json:"blockingByList,omitempty" cborgen:"blockingByList,omitempty"` + FollowedBy *string `json:"followedBy,omitempty" cborgen:"followedBy,omitempty"` + Following *string `json:"following,omitempty" cborgen:"following,omitempty"` + // knownFollowers: This property is present only in selected cases, as an optimization. + KnownFollowers *ActorDefs_KnownFollowers `json:"knownFollowers,omitempty" cborgen:"knownFollowers,omitempty"` + Muted *bool `json:"muted,omitempty" cborgen:"muted,omitempty"` + MutedByList *GraphDefs_ListViewBasic `json:"mutedByList,omitempty" cborgen:"mutedByList,omitempty"` +} diff --git a/api/bsky/actorgetPreferences.go b/api/bsky/actorgetPreferences.go new file mode 100644 index 000000000..55fc30eea --- /dev/null +++ b/api/bsky/actorgetPreferences.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.getPreferences + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ActorGetPreferences_Output is the output of a app.bsky.actor.getPreferences call. +type ActorGetPreferences_Output struct { + Preferences []ActorDefs_Preferences_Elem `json:"preferences" cborgen:"preferences"` +} + +// ActorGetPreferences calls the XRPC method "app.bsky.actor.getPreferences". +func ActorGetPreferences(ctx context.Context, c lexutil.LexClient) (*ActorGetPreferences_Output, error) { + var out ActorGetPreferences_Output + + params := map[string]interface{}{} + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.actor.getPreferences", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/actorgetProfile.go b/api/bsky/actorgetProfile.go index 4004e50a5..7550af1f3 100644 --- a/api/bsky/actorgetProfile.go +++ b/api/bsky/actorgetProfile.go @@ -1,47 +1,24 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.getProfile + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.actor.getProfile - -func init() { -} - -type ActorGetProfile_MyState struct { - LexiconTypeID string `json:"$type,omitempty"` - Follow *string `json:"follow,omitempty" cborgen:"follow"` - Member *string `json:"member,omitempty" cborgen:"member"` - Muted *bool `json:"muted,omitempty" cborgen:"muted"` -} - -type ActorGetProfile_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Avatar *string `json:"avatar,omitempty" cborgen:"avatar"` - Banner *string `json:"banner,omitempty" cborgen:"banner"` - Creator string `json:"creator" cborgen:"creator"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Description *string `json:"description,omitempty" cborgen:"description"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - FollowersCount int64 `json:"followersCount" cborgen:"followersCount"` - FollowsCount int64 `json:"followsCount" cborgen:"followsCount"` - Handle string `json:"handle" cborgen:"handle"` - MembersCount int64 `json:"membersCount" cborgen:"membersCount"` - MyState *ActorGetProfile_MyState `json:"myState,omitempty" cborgen:"myState"` - PostsCount int64 `json:"postsCount" cborgen:"postsCount"` -} +// ActorGetProfile calls the XRPC method "app.bsky.actor.getProfile". +// +// actor: Handle or DID of account to fetch profile of. +func ActorGetProfile(ctx context.Context, c lexutil.LexClient, actor string) (*ActorDefs_ProfileViewDetailed, error) { + var out ActorDefs_ProfileViewDetailed -func ActorGetProfile(ctx context.Context, c *xrpc.Client, actor string) (*ActorGetProfile_Output, error) { - var out ActorGetProfile_Output - - params := map[string]interface{}{ - "actor": actor, - } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.actor.getProfile", params, nil, &out); err != nil { + params := map[string]interface{}{} + params["actor"] = actor + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.actor.getProfile", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/actorgetProfiles.go b/api/bsky/actorgetProfiles.go new file mode 100644 index 000000000..9fbcd0606 --- /dev/null +++ b/api/bsky/actorgetProfiles.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.getProfiles + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ActorGetProfiles_Output is the output of a app.bsky.actor.getProfiles call. +type ActorGetProfiles_Output struct { + Profiles []*ActorDefs_ProfileViewDetailed `json:"profiles" cborgen:"profiles"` +} + +// ActorGetProfiles calls the XRPC method "app.bsky.actor.getProfiles". +func ActorGetProfiles(ctx context.Context, c lexutil.LexClient, actors []string) (*ActorGetProfiles_Output, error) { + var out ActorGetProfiles_Output + + params := map[string]interface{}{} + params["actors"] = actors + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.actor.getProfiles", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/actorgetSuggestions.go b/api/bsky/actorgetSuggestions.go index b5ae9d137..1647cdb0a 100644 --- a/api/bsky/actorgetSuggestions.go +++ b/api/bsky/actorgetSuggestions.go @@ -1,47 +1,35 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.getSuggestions + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.actor.getSuggestions - -func init() { -} - -type ActorGetSuggestions_Actor struct { - LexiconTypeID string `json:"$type,omitempty"` - Avatar *string `json:"avatar,omitempty" cborgen:"avatar"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Description *string `json:"description,omitempty" cborgen:"description"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt *string `json:"indexedAt,omitempty" cborgen:"indexedAt"` - MyState *ActorGetSuggestions_MyState `json:"myState,omitempty" cborgen:"myState"` -} - -type ActorGetSuggestions_MyState struct { - LexiconTypeID string `json:"$type,omitempty"` - Follow *string `json:"follow,omitempty" cborgen:"follow"` -} - +// ActorGetSuggestions_Output is the output of a app.bsky.actor.getSuggestions call. type ActorGetSuggestions_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Actors []*ActorGetSuggestions_Actor `json:"actors" cborgen:"actors"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` + Actors []*ActorDefs_ProfileView `json:"actors" cborgen:"actors"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // recId: Snowflake for this recommendation, use when submitting recommendation events. + RecId *int64 `json:"recId,omitempty" cborgen:"recId,omitempty"` } -func ActorGetSuggestions(ctx context.Context, c *xrpc.Client, cursor string, limit int64) (*ActorGetSuggestions_Output, error) { +// ActorGetSuggestions calls the XRPC method "app.bsky.actor.getSuggestions". +func ActorGetSuggestions(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*ActorGetSuggestions_Output, error) { var out ActorGetSuggestions_Output - params := map[string]interface{}{ - "cursor": cursor, - "limit": limit, + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.actor.getSuggestions", params, nil, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.actor.getSuggestions", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/actorprofile.go b/api/bsky/actorprofile.go index bb358a2f9..db6499859 100644 --- a/api/bsky/actorprofile.go +++ b/api/bsky/actorprofile.go @@ -1,20 +1,94 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.profile + +package bsky import ( - "github.com/bluesky-social/indigo/lex/util" -) + "bytes" + "encoding/json" + "fmt" + "io" -// schema: app.bsky.actor.profile + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) func init() { - util.RegisterType("app.bsky.actor.profile", &ActorProfile{}) + lexutil.RegisterType("app.bsky.actor.profile", &ActorProfile{}) } -// RECORDTYPE: ActorProfile type ActorProfile struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.profile"` - Avatar *util.Blob `json:"avatar,omitempty" cborgen:"avatar"` - Banner *util.Blob `json:"banner,omitempty" cborgen:"banner"` - Description *string `json:"description,omitempty" cborgen:"description"` - DisplayName string `json:"displayName" cborgen:"displayName"` + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.profile"` + // avatar: Small image to be displayed next to posts from account. AKA, 'profile picture' + Avatar *lexutil.LexBlob `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + // banner: Larger horizontal image to display behind profile view. + Banner *lexutil.LexBlob `json:"banner,omitempty" cborgen:"banner,omitempty"` + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + // description: Free-form profile description text. + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + DisplayName *string `json:"displayName,omitempty" cborgen:"displayName,omitempty"` + JoinedViaStarterPack *comatproto.RepoStrongRef `json:"joinedViaStarterPack,omitempty" cborgen:"joinedViaStarterPack,omitempty"` + // labels: Self-label values, specific to the Bluesky application, on the overall account. + Labels *ActorProfile_Labels `json:"labels,omitempty" cborgen:"labels,omitempty"` + PinnedPost *comatproto.RepoStrongRef `json:"pinnedPost,omitempty" cborgen:"pinnedPost,omitempty"` + // pronouns: Free-form pronouns text. + Pronouns *string `json:"pronouns,omitempty" cborgen:"pronouns,omitempty"` + Website *string `json:"website,omitempty" cborgen:"website,omitempty"` +} + +// Self-label values, specific to the Bluesky application, on the overall account. +type ActorProfile_Labels struct { + LabelDefs_SelfLabels *comatproto.LabelDefs_SelfLabels +} + +func (t *ActorProfile_Labels) MarshalJSON() ([]byte, error) { + if t.LabelDefs_SelfLabels != nil { + t.LabelDefs_SelfLabels.LexiconTypeID = "com.atproto.label.defs#selfLabels" + return json.Marshal(t.LabelDefs_SelfLabels) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ActorProfile_Labels) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return json.Unmarshal(b, t.LabelDefs_SelfLabels) + default: + return nil + } +} + +func (t *ActorProfile_Labels) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.LabelDefs_SelfLabels != nil { + return t.LabelDefs_SelfLabels.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *ActorProfile_Labels) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return t.LabelDefs_SelfLabels.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } } diff --git a/api/bsky/actorputPreferences.go b/api/bsky/actorputPreferences.go new file mode 100644 index 000000000..27fc1cb12 --- /dev/null +++ b/api/bsky/actorputPreferences.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.putPreferences + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ActorPutPreferences_Input is the input argument to a app.bsky.actor.putPreferences call. +type ActorPutPreferences_Input struct { + Preferences []ActorDefs_Preferences_Elem `json:"preferences" cborgen:"preferences"` +} + +// ActorPutPreferences calls the XRPC method "app.bsky.actor.putPreferences". +func ActorPutPreferences(ctx context.Context, c lexutil.LexClient, input *ActorPutPreferences_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.actor.putPreferences", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/actorref.go b/api/bsky/actorref.go deleted file mode 100644 index 5dc64b359..000000000 --- a/api/bsky/actorref.go +++ /dev/null @@ -1,27 +0,0 @@ -package schemagen - -// schema: app.bsky.actor.ref - -func init() { -} - -type ActorRef struct { - LexiconTypeID string `json:"$type,omitempty"` - DeclarationCid string `json:"declarationCid" cborgen:"declarationCid"` - Did string `json:"did" cborgen:"did"` -} - -type ActorRef_ViewerState struct { - LexiconTypeID string `json:"$type,omitempty"` - Muted *bool `json:"muted,omitempty" cborgen:"muted"` -} - -type ActorRef_WithInfo struct { - LexiconTypeID string `json:"$type,omitempty"` - Avatar *string `json:"avatar,omitempty" cborgen:"avatar"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` - Viewer *ActorRef_ViewerState `json:"viewer,omitempty" cborgen:"viewer"` -} diff --git a/api/bsky/actorsearch.go b/api/bsky/actorsearch.go deleted file mode 100644 index 829bee557..000000000 --- a/api/bsky/actorsearch.go +++ /dev/null @@ -1,44 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.actor.search - -func init() { -} - -type ActorSearch_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Users []*ActorSearch_User `json:"users" cborgen:"users"` -} - -type ActorSearch_User struct { - LexiconTypeID string `json:"$type,omitempty"` - Avatar *string `json:"avatar,omitempty" cborgen:"avatar"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Description *string `json:"description,omitempty" cborgen:"description"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt *string `json:"indexedAt,omitempty" cborgen:"indexedAt"` -} - -func ActorSearch(ctx context.Context, c *xrpc.Client, before string, limit int64, term string) (*ActorSearch_Output, error) { - var out ActorSearch_Output - - params := map[string]interface{}{ - "before": before, - "limit": limit, - "term": term, - } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.actor.search", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/actorsearchActors.go b/api/bsky/actorsearchActors.go new file mode 100644 index 000000000..6673c5e41 --- /dev/null +++ b/api/bsky/actorsearchActors.go @@ -0,0 +1,44 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.searchActors + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ActorSearchActors_Output is the output of a app.bsky.actor.searchActors call. +type ActorSearchActors_Output struct { + Actors []*ActorDefs_ProfileView `json:"actors" cborgen:"actors"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// ActorSearchActors calls the XRPC method "app.bsky.actor.searchActors". +// +// q: Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. +// term: DEPRECATED: use 'q' instead. +func ActorSearchActors(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, q string, term string) (*ActorSearchActors_Output, error) { + var out ActorSearchActors_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if q != "" { + params["q"] = q + } + if term != "" { + params["term"] = term + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.actor.searchActors", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/actorsearchActorsTypeahead.go b/api/bsky/actorsearchActorsTypeahead.go new file mode 100644 index 000000000..7b7bea03c --- /dev/null +++ b/api/bsky/actorsearchActorsTypeahead.go @@ -0,0 +1,40 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.searchActorsTypeahead + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ActorSearchActorsTypeahead_Output is the output of a app.bsky.actor.searchActorsTypeahead call. +type ActorSearchActorsTypeahead_Output struct { + Actors []*ActorDefs_ProfileViewBasic `json:"actors" cborgen:"actors"` +} + +// ActorSearchActorsTypeahead calls the XRPC method "app.bsky.actor.searchActorsTypeahead". +// +// q: Search query prefix; not a full query string. +// term: DEPRECATED: use 'q' instead. +func ActorSearchActorsTypeahead(ctx context.Context, c lexutil.LexClient, limit int64, q string, term string) (*ActorSearchActorsTypeahead_Output, error) { + var out ActorSearchActorsTypeahead_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if q != "" { + params["q"] = q + } + if term != "" { + params["term"] = term + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.actor.searchActorsTypeahead", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/actorsearchTypeahead.go b/api/bsky/actorsearchTypeahead.go deleted file mode 100644 index 12f828b3d..000000000 --- a/api/bsky/actorsearchTypeahead.go +++ /dev/null @@ -1,40 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.actor.searchTypeahead - -func init() { -} - -type ActorSearchTypeahead_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Users []*ActorSearchTypeahead_User `json:"users" cborgen:"users"` -} - -type ActorSearchTypeahead_User struct { - LexiconTypeID string `json:"$type,omitempty"` - Avatar *string `json:"avatar,omitempty" cborgen:"avatar"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` -} - -func ActorSearchTypeahead(ctx context.Context, c *xrpc.Client, limit int64, term string) (*ActorSearchTypeahead_Output, error) { - var out ActorSearchTypeahead_Output - - params := map[string]interface{}{ - "limit": limit, - "term": term, - } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.actor.searchTypeahead", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/actorstatus.go b/api/bsky/actorstatus.go new file mode 100644 index 000000000..414d00c0b --- /dev/null +++ b/api/bsky/actorstatus.go @@ -0,0 +1,85 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.actor.status + +package bsky + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func init() { + lexutil.RegisterType("app.bsky.actor.status", &ActorStatus{}) +} + +type ActorStatus struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.actor.status"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // durationMinutes: The duration of the status in minutes. Applications can choose to impose minimum and maximum limits. + DurationMinutes *int64 `json:"durationMinutes,omitempty" cborgen:"durationMinutes,omitempty"` + // embed: An optional embed associated with the status. + Embed *ActorStatus_Embed `json:"embed,omitempty" cborgen:"embed,omitempty"` + // status: The status for the account. + Status string `json:"status" cborgen:"status"` +} + +// An optional embed associated with the status. +type ActorStatus_Embed struct { + EmbedExternal *EmbedExternal +} + +func (t *ActorStatus_Embed) MarshalJSON() ([]byte, error) { + if t.EmbedExternal != nil { + t.EmbedExternal.LexiconTypeID = "app.bsky.embed.external" + return json.Marshal(t.EmbedExternal) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ActorStatus_Embed) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.external": + t.EmbedExternal = new(EmbedExternal) + return json.Unmarshal(b, t.EmbedExternal) + default: + return nil + } +} + +func (t *ActorStatus_Embed) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.EmbedExternal != nil { + return t.EmbedExternal.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *ActorStatus_Embed) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.external": + t.EmbedExternal = new(EmbedExternal) + return t.EmbedExternal.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } +} diff --git a/api/bsky/actorupdateProfile.go b/api/bsky/actorupdateProfile.go deleted file mode 100644 index c8b84bdb2..000000000 --- a/api/bsky/actorupdateProfile.go +++ /dev/null @@ -1,38 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.actor.updateProfile - -func init() { -} - -type ActorUpdateProfile_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Avatar *util.Blob `json:"avatar,omitempty" cborgen:"avatar"` - Banner *util.Blob `json:"banner,omitempty" cborgen:"banner"` - Description *string `json:"description,omitempty" cborgen:"description"` - Did *string `json:"did,omitempty" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` -} - -type ActorUpdateProfile_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid string `json:"cid" cborgen:"cid"` - Record util.LexiconTypeDecoder `json:"record" cborgen:"record"` - Uri string `json:"uri" cborgen:"uri"` -} - -func ActorUpdateProfile(ctx context.Context, c *xrpc.Client, input *ActorUpdateProfile_Input) (*ActorUpdateProfile_Output, error) { - var out ActorUpdateProfile_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "app.bsky.actor.updateProfile", nil, input, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/ageassurancebegin.go b/api/bsky/ageassurancebegin.go new file mode 100644 index 000000000..d6bb07f5a --- /dev/null +++ b/api/bsky/ageassurancebegin.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.ageassurance.begin + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AgeassuranceBegin_Input is the input argument to a app.bsky.ageassurance.begin call. +type AgeassuranceBegin_Input struct { + // countryCode: An ISO 3166-1 alpha-2 code of the user's location. + CountryCode string `json:"countryCode" cborgen:"countryCode"` + // email: The user's email address to receive Age Assurance instructions. + Email string `json:"email" cborgen:"email"` + // language: The user's preferred language for communication during the Age Assurance process. + Language string `json:"language" cborgen:"language"` + // regionCode: An optional ISO 3166-2 code of the user's region or state within the country. + RegionCode *string `json:"regionCode,omitempty" cborgen:"regionCode,omitempty"` +} + +// AgeassuranceBegin calls the XRPC method "app.bsky.ageassurance.begin". +func AgeassuranceBegin(ctx context.Context, c lexutil.LexClient, input *AgeassuranceBegin_Input) (*AgeassuranceDefs_State, error) { + var out AgeassuranceDefs_State + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.ageassurance.begin", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/ageassurancedefs.go b/api/bsky/ageassurancedefs.go new file mode 100644 index 000000000..524088b94 --- /dev/null +++ b/api/bsky/ageassurancedefs.go @@ -0,0 +1,219 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.ageassurance.defs + +package bsky + +import ( + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AgeassuranceDefs_Config is a "config" in the app.bsky.ageassurance.defs schema. +type AgeassuranceDefs_Config struct { + // regions: The per-region Age Assurance configuration. + Regions []*AgeassuranceDefs_ConfigRegion `json:"regions" cborgen:"regions"` +} + +// AgeassuranceDefs_ConfigRegion is a "configRegion" in the app.bsky.ageassurance.defs schema. +// +// The Age Assurance configuration for a specific region. +type AgeassuranceDefs_ConfigRegion struct { + // countryCode: The ISO 3166-1 alpha-2 country code this configuration applies to. + CountryCode string `json:"countryCode" cborgen:"countryCode"` + // regionCode: The ISO 3166-2 region code this configuration applies to. If omitted, the configuration applies to the entire country. + RegionCode *string `json:"regionCode,omitempty" cborgen:"regionCode,omitempty"` + // rules: The ordered list of Age Assurance rules that apply to this region. Rules should be applied in order, and the first matching rule determines the access level granted. The rules array should always include a default rule as the last item. + Rules []*AgeassuranceDefs_ConfigRegion_Rules_Elem `json:"rules" cborgen:"rules"` +} + +// AgeassuranceDefs_ConfigRegionRuleDefault is a "configRegionRuleDefault" in the app.bsky.ageassurance.defs schema. +// +// Age Assurance rule that applies by default. +type AgeassuranceDefs_ConfigRegionRuleDefault struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.ageassurance.defs#configRegionRuleDefault"` + Access *string `json:"access" cborgen:"access"` +} + +// AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan is a "configRegionRuleIfAccountNewerThan" in the app.bsky.ageassurance.defs schema. +// +// Age Assurance rule that applies if the account is equal-to or newer than a certain date. +type AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.ageassurance.defs#configRegionRuleIfAccountNewerThan"` + Access *string `json:"access" cborgen:"access"` + // date: The date threshold as a datetime string. + Date string `json:"date" cborgen:"date"` +} + +// AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan is a "configRegionRuleIfAccountOlderThan" in the app.bsky.ageassurance.defs schema. +// +// Age Assurance rule that applies if the account is older than a certain date. +type AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.ageassurance.defs#configRegionRuleIfAccountOlderThan"` + Access *string `json:"access" cborgen:"access"` + // date: The date threshold as a datetime string. + Date string `json:"date" cborgen:"date"` +} + +// AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge is a "configRegionRuleIfAssuredOverAge" in the app.bsky.ageassurance.defs schema. +// +// Age Assurance rule that applies if the user has been assured to be equal-to or over a certain age. +type AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.ageassurance.defs#configRegionRuleIfAssuredOverAge"` + Access *string `json:"access" cborgen:"access"` + // age: The age threshold as a whole integer. + Age int64 `json:"age" cborgen:"age"` +} + +// AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge is a "configRegionRuleIfAssuredUnderAge" in the app.bsky.ageassurance.defs schema. +// +// Age Assurance rule that applies if the user has been assured to be under a certain age. +type AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.ageassurance.defs#configRegionRuleIfAssuredUnderAge"` + Access *string `json:"access" cborgen:"access"` + // age: The age threshold as a whole integer. + Age int64 `json:"age" cborgen:"age"` +} + +// AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge is a "configRegionRuleIfDeclaredOverAge" in the app.bsky.ageassurance.defs schema. +// +// Age Assurance rule that applies if the user has declared themselves equal-to or over a certain age. +type AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.ageassurance.defs#configRegionRuleIfDeclaredOverAge"` + Access *string `json:"access" cborgen:"access"` + // age: The age threshold as a whole integer. + Age int64 `json:"age" cborgen:"age"` +} + +// AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge is a "configRegionRuleIfDeclaredUnderAge" in the app.bsky.ageassurance.defs schema. +// +// Age Assurance rule that applies if the user has declared themselves under a certain age. +type AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.ageassurance.defs#configRegionRuleIfDeclaredUnderAge"` + Access *string `json:"access" cborgen:"access"` + // age: The age threshold as a whole integer. + Age int64 `json:"age" cborgen:"age"` +} + +type AgeassuranceDefs_ConfigRegion_Rules_Elem struct { + AgeassuranceDefs_ConfigRegionRuleDefault *AgeassuranceDefs_ConfigRegionRuleDefault + AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge *AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge + AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge *AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge + AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge *AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge + AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge *AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge + AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan *AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan + AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan *AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan +} + +func (t *AgeassuranceDefs_ConfigRegion_Rules_Elem) MarshalJSON() ([]byte, error) { + if t.AgeassuranceDefs_ConfigRegionRuleDefault != nil { + t.AgeassuranceDefs_ConfigRegionRuleDefault.LexiconTypeID = "app.bsky.ageassurance.defs#configRegionRuleDefault" + return json.Marshal(t.AgeassuranceDefs_ConfigRegionRuleDefault) + } + if t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge != nil { + t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge.LexiconTypeID = "app.bsky.ageassurance.defs#configRegionRuleIfDeclaredOverAge" + return json.Marshal(t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge) + } + if t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge != nil { + t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge.LexiconTypeID = "app.bsky.ageassurance.defs#configRegionRuleIfDeclaredUnderAge" + return json.Marshal(t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge) + } + if t.AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge != nil { + t.AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge.LexiconTypeID = "app.bsky.ageassurance.defs#configRegionRuleIfAssuredOverAge" + return json.Marshal(t.AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge) + } + if t.AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge != nil { + t.AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge.LexiconTypeID = "app.bsky.ageassurance.defs#configRegionRuleIfAssuredUnderAge" + return json.Marshal(t.AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge) + } + if t.AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan != nil { + t.AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan.LexiconTypeID = "app.bsky.ageassurance.defs#configRegionRuleIfAccountNewerThan" + return json.Marshal(t.AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan) + } + if t.AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan != nil { + t.AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan.LexiconTypeID = "app.bsky.ageassurance.defs#configRegionRuleIfAccountOlderThan" + return json.Marshal(t.AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *AgeassuranceDefs_ConfigRegion_Rules_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.ageassurance.defs#configRegionRuleDefault": + t.AgeassuranceDefs_ConfigRegionRuleDefault = new(AgeassuranceDefs_ConfigRegionRuleDefault) + return json.Unmarshal(b, t.AgeassuranceDefs_ConfigRegionRuleDefault) + case "app.bsky.ageassurance.defs#configRegionRuleIfDeclaredOverAge": + t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge = new(AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge) + return json.Unmarshal(b, t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredOverAge) + case "app.bsky.ageassurance.defs#configRegionRuleIfDeclaredUnderAge": + t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge = new(AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge) + return json.Unmarshal(b, t.AgeassuranceDefs_ConfigRegionRuleIfDeclaredUnderAge) + case "app.bsky.ageassurance.defs#configRegionRuleIfAssuredOverAge": + t.AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge = new(AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge) + return json.Unmarshal(b, t.AgeassuranceDefs_ConfigRegionRuleIfAssuredOverAge) + case "app.bsky.ageassurance.defs#configRegionRuleIfAssuredUnderAge": + t.AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge = new(AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge) + return json.Unmarshal(b, t.AgeassuranceDefs_ConfigRegionRuleIfAssuredUnderAge) + case "app.bsky.ageassurance.defs#configRegionRuleIfAccountNewerThan": + t.AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan = new(AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan) + return json.Unmarshal(b, t.AgeassuranceDefs_ConfigRegionRuleIfAccountNewerThan) + case "app.bsky.ageassurance.defs#configRegionRuleIfAccountOlderThan": + t.AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan = new(AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan) + return json.Unmarshal(b, t.AgeassuranceDefs_ConfigRegionRuleIfAccountOlderThan) + default: + return nil + } +} + +// AgeassuranceDefs_Event is a "event" in the app.bsky.ageassurance.defs schema. +// +// Object used to store Age Assurance data in stash. +type AgeassuranceDefs_Event struct { + // access: The access level granted based on Age Assurance data we've processed. + Access string `json:"access" cborgen:"access"` + // attemptId: The unique identifier for this instance of the Age Assurance flow, in UUID format. + AttemptId string `json:"attemptId" cborgen:"attemptId"` + // completeIp: The IP address used when completing the Age Assurance flow. + CompleteIp *string `json:"completeIp,omitempty" cborgen:"completeIp,omitempty"` + // completeUa: The user agent used when completing the Age Assurance flow. + CompleteUa *string `json:"completeUa,omitempty" cborgen:"completeUa,omitempty"` + // countryCode: The ISO 3166-1 alpha-2 country code provided when beginning the Age Assurance flow. + CountryCode string `json:"countryCode" cborgen:"countryCode"` + // createdAt: The date and time of this write operation. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // email: The email used for Age Assurance. + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + // initIp: The IP address used when initiating the Age Assurance flow. + InitIp *string `json:"initIp,omitempty" cborgen:"initIp,omitempty"` + // initUa: The user agent used when initiating the Age Assurance flow. + InitUa *string `json:"initUa,omitempty" cborgen:"initUa,omitempty"` + // regionCode: The ISO 3166-2 region code provided when beginning the Age Assurance flow. + RegionCode *string `json:"regionCode,omitempty" cborgen:"regionCode,omitempty"` + // status: The status of the Age Assurance process. + Status string `json:"status" cborgen:"status"` +} + +// AgeassuranceDefs_State is a "state" in the app.bsky.ageassurance.defs schema. +// +// The user's computed Age Assurance state. +type AgeassuranceDefs_State struct { + Access *string `json:"access" cborgen:"access"` + // lastInitiatedAt: The timestamp when this state was last updated. + LastInitiatedAt *string `json:"lastInitiatedAt,omitempty" cborgen:"lastInitiatedAt,omitempty"` + Status *string `json:"status" cborgen:"status"` +} + +// AgeassuranceDefs_StateMetadata is a "stateMetadata" in the app.bsky.ageassurance.defs schema. +// +// Additional metadata needed to compute Age Assurance state client-side. +type AgeassuranceDefs_StateMetadata struct { + // accountCreatedAt: The account creation timestamp. + AccountCreatedAt *string `json:"accountCreatedAt,omitempty" cborgen:"accountCreatedAt,omitempty"` +} diff --git a/api/bsky/ageassurancegetConfig.go b/api/bsky/ageassurancegetConfig.go new file mode 100644 index 000000000..77f50b020 --- /dev/null +++ b/api/bsky/ageassurancegetConfig.go @@ -0,0 +1,21 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.ageassurance.getConfig + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AgeassuranceGetConfig calls the XRPC method "app.bsky.ageassurance.getConfig". +func AgeassuranceGetConfig(ctx context.Context, c lexutil.LexClient) (*AgeassuranceDefs_Config, error) { + var out AgeassuranceDefs_Config + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.ageassurance.getConfig", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/ageassurancegetState.go b/api/bsky/ageassurancegetState.go new file mode 100644 index 000000000..d6aac8f08 --- /dev/null +++ b/api/bsky/ageassurancegetState.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.ageassurance.getState + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// AgeassuranceGetState_Output is the output of a app.bsky.ageassurance.getState call. +type AgeassuranceGetState_Output struct { + Metadata *AgeassuranceDefs_StateMetadata `json:"metadata" cborgen:"metadata"` + State *AgeassuranceDefs_State `json:"state" cborgen:"state"` +} + +// AgeassuranceGetState calls the XRPC method "app.bsky.ageassurance.getState". +func AgeassuranceGetState(ctx context.Context, c lexutil.LexClient, countryCode string, regionCode string) (*AgeassuranceGetState_Output, error) { + var out AgeassuranceGetState_Output + + params := map[string]interface{}{} + params["countryCode"] = countryCode + if regionCode != "" { + params["regionCode"] = regionCode + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.ageassurance.getState", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/bookmarkcreateBookmark.go b/api/bsky/bookmarkcreateBookmark.go new file mode 100644 index 000000000..5be7cb311 --- /dev/null +++ b/api/bsky/bookmarkcreateBookmark.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.bookmark.createBookmark + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// BookmarkCreateBookmark_Input is the input argument to a app.bsky.bookmark.createBookmark call. +type BookmarkCreateBookmark_Input struct { + Cid string `json:"cid" cborgen:"cid"` + Uri string `json:"uri" cborgen:"uri"` +} + +// BookmarkCreateBookmark calls the XRPC method "app.bsky.bookmark.createBookmark". +func BookmarkCreateBookmark(ctx context.Context, c lexutil.LexClient, input *BookmarkCreateBookmark_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.bookmark.createBookmark", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/bookmarkdefs.go b/api/bsky/bookmarkdefs.go new file mode 100644 index 000000000..5cb29cfb1 --- /dev/null +++ b/api/bsky/bookmarkdefs.go @@ -0,0 +1,72 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.bookmark.defs + +package bsky + +import ( + "encoding/json" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// BookmarkDefs_Bookmark is a "bookmark" in the app.bsky.bookmark.defs schema. +// +// Object used to store bookmark data in stash. +type BookmarkDefs_Bookmark struct { + // subject: A strong ref to the record to be bookmarked. Currently, only `app.bsky.feed.post` records are supported. + Subject *comatproto.RepoStrongRef `json:"subject" cborgen:"subject"` +} + +// BookmarkDefs_BookmarkView is a "bookmarkView" in the app.bsky.bookmark.defs schema. +type BookmarkDefs_BookmarkView struct { + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + Item *BookmarkDefs_BookmarkView_Item `json:"item" cborgen:"item"` + // subject: A strong ref to the bookmarked record. + Subject *comatproto.RepoStrongRef `json:"subject" cborgen:"subject"` +} + +type BookmarkDefs_BookmarkView_Item struct { + FeedDefs_BlockedPost *FeedDefs_BlockedPost + FeedDefs_NotFoundPost *FeedDefs_NotFoundPost + FeedDefs_PostView *FeedDefs_PostView +} + +func (t *BookmarkDefs_BookmarkView_Item) MarshalJSON() ([]byte, error) { + if t.FeedDefs_BlockedPost != nil { + t.FeedDefs_BlockedPost.LexiconTypeID = "app.bsky.feed.defs#blockedPost" + return json.Marshal(t.FeedDefs_BlockedPost) + } + if t.FeedDefs_NotFoundPost != nil { + t.FeedDefs_NotFoundPost.LexiconTypeID = "app.bsky.feed.defs#notFoundPost" + return json.Marshal(t.FeedDefs_NotFoundPost) + } + if t.FeedDefs_PostView != nil { + t.FeedDefs_PostView.LexiconTypeID = "app.bsky.feed.defs#postView" + return json.Marshal(t.FeedDefs_PostView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *BookmarkDefs_BookmarkView_Item) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.defs#blockedPost": + t.FeedDefs_BlockedPost = new(FeedDefs_BlockedPost) + return json.Unmarshal(b, t.FeedDefs_BlockedPost) + case "app.bsky.feed.defs#notFoundPost": + t.FeedDefs_NotFoundPost = new(FeedDefs_NotFoundPost) + return json.Unmarshal(b, t.FeedDefs_NotFoundPost) + case "app.bsky.feed.defs#postView": + t.FeedDefs_PostView = new(FeedDefs_PostView) + return json.Unmarshal(b, t.FeedDefs_PostView) + default: + return nil + } +} diff --git a/api/bsky/bookmarkdeleteBookmark.go b/api/bsky/bookmarkdeleteBookmark.go new file mode 100644 index 000000000..3074cca08 --- /dev/null +++ b/api/bsky/bookmarkdeleteBookmark.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.bookmark.deleteBookmark + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// BookmarkDeleteBookmark_Input is the input argument to a app.bsky.bookmark.deleteBookmark call. +type BookmarkDeleteBookmark_Input struct { + Uri string `json:"uri" cborgen:"uri"` +} + +// BookmarkDeleteBookmark calls the XRPC method "app.bsky.bookmark.deleteBookmark". +func BookmarkDeleteBookmark(ctx context.Context, c lexutil.LexClient, input *BookmarkDeleteBookmark_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.bookmark.deleteBookmark", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/bookmarkgetBookmarks.go b/api/bsky/bookmarkgetBookmarks.go new file mode 100644 index 000000000..6dd05c7c4 --- /dev/null +++ b/api/bsky/bookmarkgetBookmarks.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.bookmark.getBookmarks + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// BookmarkGetBookmarks_Output is the output of a app.bsky.bookmark.getBookmarks call. +type BookmarkGetBookmarks_Output struct { + Bookmarks []*BookmarkDefs_BookmarkView `json:"bookmarks" cborgen:"bookmarks"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// BookmarkGetBookmarks calls the XRPC method "app.bsky.bookmark.getBookmarks". +func BookmarkGetBookmarks(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*BookmarkGetBookmarks_Output, error) { + var out BookmarkGetBookmarks_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.bookmark.getBookmarks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/cbor_gen.go b/api/bsky/cbor_gen.go index a41ccf6f3..e8de2eda0 100644 --- a/api/bsky/cbor_gen.go +++ b/api/bsky/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package schemagen +package bsky import ( "fmt" @@ -8,7 +8,7 @@ import ( "math" "sort" - schemagen "github.com/bluesky-social/indigo/api/atproto" + atproto "github.com/bluesky-social/indigo/api/atproto" util "github.com/bluesky-social/indigo/lex/util" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -27,130 +27,289 @@ func (t *FeedPost) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) + fieldCount := 10 - if _, err := cw.Write([]byte{166}); err != nil { + if t.Embed == nil { + fieldCount-- + } + + if t.Entities == nil { + fieldCount-- + } + + if t.Facets == nil { + fieldCount-- + } + + if t.Labels == nil { + fieldCount-- + } + + if t.Langs == nil { + fieldCount-- + } + + if t.Reply == nil { + fieldCount-- + } + + if t.Tags == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } + // t.Tags ([]string) (slice) + if t.Tags != nil { + + if len("tags") > 1000000 { + return xerrors.Errorf("Value in field \"tags\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("tags"))); err != nil { + return err + } + if _, err := cw.WriteString(string("tags")); err != nil { + return err + } + + if len(t.Tags) > 8192 { + return xerrors.Errorf("Slice value in field t.Tags was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Tags))); err != nil { + return err + } + for _, v := range t.Tags { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } + // t.Text (string) (string) - if len("text") > cbg.MaxLength { + if len("text") > 1000000 { return xerrors.Errorf("Value in field \"text\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("text"))); err != nil { return err } - if _, err := io.WriteString(w, string("text")); err != nil { + if _, err := cw.WriteString(string("text")); err != nil { return err } - if len(t.Text) > cbg.MaxLength { + if len(t.Text) > 1000000 { return xerrors.Errorf("Value in field t.Text was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Text))); err != nil { return err } - if _, err := io.WriteString(w, string(t.Text)); err != nil { + if _, err := cw.WriteString(string(t.Text)); err != nil { return err } // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("$type")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.post"))); err != nil { return err } - if _, err := io.WriteString(w, string("app.bsky.feed.post")); err != nil { + if _, err := cw.WriteString(string("app.bsky.feed.post")); err != nil { return err } - // t.Embed (schemagen.FeedPost_Embed) (struct) - if len("embed") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"embed\" was too long") - } + // t.Embed (bsky.FeedPost_Embed) (struct) + if t.Embed != nil { - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("embed"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("embed")); err != nil { - return err - } + if len("embed") > 1000000 { + return xerrors.Errorf("Value in field \"embed\" was too long") + } - if err := t.Embed.MarshalCBOR(cw); err != nil { - return err - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("embed"))); err != nil { + return err + } + if _, err := cw.WriteString(string("embed")); err != nil { + return err + } - // t.Reply (schemagen.FeedPost_ReplyRef) (struct) - if len("reply") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"reply\" was too long") + if err := t.Embed.MarshalCBOR(cw); err != nil { + return err + } } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("reply"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("reply")); err != nil { - return err - } + // t.Langs ([]string) (slice) + if t.Langs != nil { - if err := t.Reply.MarshalCBOR(cw); err != nil { - return err - } + if len("langs") > 1000000 { + return xerrors.Errorf("Value in field \"langs\" was too long") + } - // t.Entities ([]*schemagen.FeedPost_Entity) (slice) - if len("entities") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"entities\" was too long") - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("langs"))); err != nil { + return err + } + if _, err := cw.WriteString(string("langs")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("entities"))); err != nil { - return err + if len(t.Langs) > 8192 { + return xerrors.Errorf("Slice value in field t.Langs was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Langs))); err != nil { + return err + } + for _, v := range t.Langs { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } } - if _, err := io.WriteString(w, string("entities")); err != nil { - return err + + // t.Reply (bsky.FeedPost_ReplyRef) (struct) + if t.Reply != nil { + + if len("reply") > 1000000 { + return xerrors.Errorf("Value in field \"reply\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("reply"))); err != nil { + return err + } + if _, err := cw.WriteString(string("reply")); err != nil { + return err + } + + if err := t.Reply.MarshalCBOR(cw); err != nil { + return err + } } - if len(t.Entities) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field t.Entities was too long") + // t.Facets ([]*bsky.RichtextFacet) (slice) + if t.Facets != nil { + + if len("facets") > 1000000 { + return xerrors.Errorf("Value in field \"facets\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("facets"))); err != nil { + return err + } + if _, err := cw.WriteString(string("facets")); err != nil { + return err + } + + if len(t.Facets) > 8192 { + return xerrors.Errorf("Slice value in field t.Facets was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Facets))); err != nil { + return err + } + for _, v := range t.Facets { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } } - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Entities))); err != nil { - return err + // t.Labels (bsky.FeedPost_Labels) (struct) + if t.Labels != nil { + + if len("labels") > 1000000 { + return xerrors.Errorf("Value in field \"labels\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("labels"))); err != nil { + return err + } + if _, err := cw.WriteString(string("labels")); err != nil { + return err + } + + if err := t.Labels.MarshalCBOR(cw); err != nil { + return err + } } - for _, v := range t.Entities { - if err := v.MarshalCBOR(cw); err != nil { + + // t.Entities ([]*bsky.FeedPost_Entity) (slice) + if t.Entities != nil { + + if len("entities") > 1000000 { + return xerrors.Errorf("Value in field \"entities\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("entities"))); err != nil { + return err + } + if _, err := cw.WriteString(string("entities")); err != nil { + return err + } + + if len(t.Entities) > 8192 { + return xerrors.Errorf("Slice value in field t.Entities was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Entities))); err != nil { return err } + for _, v := range t.Entities { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } } // t.CreatedAt (string) (string) - if len("createdAt") > cbg.MaxLength { + if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { return err } - if _, err := io.WriteString(w, string("createdAt")); err != nil { + if _, err := cw.WriteString(string("createdAt")); err != nil { return err } - if len(t.CreatedAt) > cbg.MaxLength { + if len(t.CreatedAt) > 1000000 { return xerrors.Errorf("Value in field t.CreatedAt was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { return err } - if _, err := io.WriteString(w, string(t.CreatedAt)); err != nil { + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } return nil @@ -179,26 +338,69 @@ func (t *FeedPost) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedPost: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Tags ([]string) (slice) + case "tags": - { - sval, err := cbg.ReadString(cr) + maj, extra, err = cr.ReadHeader() if err != nil { return err } - name = string(sval) - } + if extra > 8192 { + return fmt.Errorf("t.Tags: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Tags = make([]string, extra) + } - switch name { - // t.Text (string) (string) + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Tags[i] = string(sval) + } + + } + } + // t.Text (string) (string) case "text": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -209,14 +411,14 @@ func (t *FeedPost) UnmarshalCBOR(r io.Reader) (err error) { case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.LexiconTypeID = string(sval) } - // t.Embed (schemagen.FeedPost_Embed) (struct) + // t.Embed (bsky.FeedPost_Embed) (struct) case "embed": { @@ -236,7 +438,47 @@ func (t *FeedPost) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.Reply (schemagen.FeedPost_ReplyRef) (struct) + // t.Langs ([]string) (slice) + case "langs": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Langs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Langs = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Langs[i] = string(sval) + } + + } + } + // t.Reply (bsky.FeedPost_ReplyRef) (struct) case "reply": { @@ -256,16 +498,16 @@ func (t *FeedPost) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.Entities ([]*schemagen.FeedPost_Entity) (slice) - case "entities": + // t.Facets ([]*bsky.RichtextFacet) (slice) + case "facets": maj, extra, err = cr.ReadHeader() if err != nil { return err } - if extra > cbg.MaxLength { - return fmt.Errorf("t.Entities: array too large (%d)", extra) + if extra > 8192 { + return fmt.Errorf("t.Facets: array too large (%d)", extra) } if maj != cbg.MajArray { @@ -273,24 +515,112 @@ func (t *FeedPost) UnmarshalCBOR(r io.Reader) (err error) { } if extra > 0 { - t.Entities = make([]*FeedPost_Entity, extra) + t.Facets = make([]*RichtextFacet, extra) } for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Facets[i] = new(RichtextFacet) + if err := t.Facets[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Facets[i] pointer: %w", err) + } + } - var v FeedPost_Entity - if err := v.UnmarshalCBOR(cr); err != nil { + } + + } + } + // t.Labels (bsky.FeedPost_Labels) (struct) + case "labels": + + { + + b, err := cr.ReadByte() + if err != nil { return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Labels = new(FeedPost_Labels) + if err := t.Labels.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Labels pointer: %w", err) + } + } + + } + // t.Entities ([]*bsky.FeedPost_Entity) (slice) + case "entities": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Entities: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } - t.Entities[i] = &v + if extra > 0 { + t.Entities = make([]*FeedPost_Entity, extra) } + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Entities[i] = new(FeedPost_Entity) + if err := t.Entities[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Entities[i] pointer: %w", err) + } + } + + } + + } + } // t.CreatedAt (string) (string) case "createdAt": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -300,7 +630,9 @@ func (t *FeedPost) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } @@ -313,39 +645,63 @@ func (t *FeedRepost) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) + fieldCount := 4 - if _, err := cw.Write([]byte{163}); err != nil { + if t.Via == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } + // t.Via (atproto.RepoStrongRef) (struct) + if t.Via != nil { + + if len("via") > 1000000 { + return xerrors.Errorf("Value in field \"via\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("via"))); err != nil { + return err + } + if _, err := cw.WriteString(string("via")); err != nil { + return err + } + + if err := t.Via.MarshalCBOR(cw); err != nil { + return err + } + } + // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("$type")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.repost"))); err != nil { return err } - if _, err := io.WriteString(w, string("app.bsky.feed.repost")); err != nil { + if _, err := cw.WriteString(string("app.bsky.feed.repost")); err != nil { return err } - // t.Subject (schemagen.RepoStrongRef) (struct) - if len("subject") > cbg.MaxLength { + // t.Subject (atproto.RepoStrongRef) (struct) + if len("subject") > 1000000 { return xerrors.Errorf("Value in field \"subject\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { return err } - if _, err := io.WriteString(w, string("subject")); err != nil { + if _, err := cw.WriteString(string("subject")); err != nil { return err } @@ -354,25 +710,25 @@ func (t *FeedRepost) MarshalCBOR(w io.Writer) error { } // t.CreatedAt (string) (string) - if len("createdAt") > cbg.MaxLength { + if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { return err } - if _, err := io.WriteString(w, string("createdAt")); err != nil { + if _, err := cw.WriteString(string("createdAt")); err != nil { return err } - if len(t.CreatedAt) > cbg.MaxLength { + if len(t.CreatedAt) > 1000000 { return xerrors.Errorf("Value in field t.CreatedAt was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { return err } - if _, err := io.WriteString(w, string(t.CreatedAt)); err != nil { + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } return nil @@ -401,33 +757,56 @@ func (t *FeedRepost) UnmarshalCBOR(r io.Reader) (err error) { return fmt.Errorf("FeedRepost: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.LexiconTypeID (string) (string) + switch string(nameBuf[:nameLen]) { + // t.Via (atproto.RepoStrongRef) (struct) + case "via": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Via = new(atproto.RepoStrongRef) + if err := t.Via.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Via pointer: %w", err) + } + } + + } + // t.LexiconTypeID (string) (string) case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.LexiconTypeID = string(sval) } - // t.Subject (schemagen.RepoStrongRef) (struct) + // t.Subject (atproto.RepoStrongRef) (struct) case "subject": { @@ -440,7 +819,7 @@ func (t *FeedRepost) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Subject = new(schemagen.RepoStrongRef) + t.Subject = new(atproto.RepoStrongRef) if err := t.Subject.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Subject pointer: %w", err) } @@ -451,7 +830,7 @@ func (t *FeedRepost) UnmarshalCBOR(r io.Reader) (err error) { case "createdAt": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } @@ -461,13 +840,15 @@ func (t *FeedRepost) UnmarshalCBOR(r io.Reader) (err error) { default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *FeedTrend) MarshalCBOR(w io.Writer) error { +func (t *FeedPost_Entity) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -479,68 +860,72 @@ func (t *FeedTrend) MarshalCBOR(w io.Writer) error { return err } - // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"$type\" was too long") + // t.Type (string) (string) + if len("type") > 1000000 { + return xerrors.Errorf("Value in field \"type\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("type"))); err != nil { return err } - if _, err := io.WriteString(w, string("$type")); err != nil { + if _, err := cw.WriteString(string("type")); err != nil { return err } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.trend"))); err != nil { + if len(t.Type) > 1000000 { + return xerrors.Errorf("Value in field t.Type was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Type))); err != nil { return err } - if _, err := io.WriteString(w, string("app.bsky.feed.trend")); err != nil { + if _, err := cw.WriteString(string(t.Type)); err != nil { return err } - // t.Subject (schemagen.RepoStrongRef) (struct) - if len("subject") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"subject\" was too long") + // t.Index (bsky.FeedPost_TextSlice) (struct) + if len("index") > 1000000 { + return xerrors.Errorf("Value in field \"index\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("index"))); err != nil { return err } - if _, err := io.WriteString(w, string("subject")); err != nil { + if _, err := cw.WriteString(string("index")); err != nil { return err } - if err := t.Subject.MarshalCBOR(cw); err != nil { + if err := t.Index.MarshalCBOR(cw); err != nil { return err } - // t.CreatedAt (string) (string) - if len("createdAt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"createdAt\" was too long") + // t.Value (string) (string) + if len("value") > 1000000 { + return xerrors.Errorf("Value in field \"value\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("value"))); err != nil { return err } - if _, err := io.WriteString(w, string("createdAt")); err != nil { + if _, err := cw.WriteString(string("value")); err != nil { return err } - if len(t.CreatedAt) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.CreatedAt was too long") + if len(t.Value) > 1000000 { + return xerrors.Errorf("Value in field t.Value was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil { return err } - if _, err := io.WriteString(w, string(t.CreatedAt)); err != nil { + if _, err := cw.WriteString(string(t.Value)); err != nil { return err } return nil } -func (t *FeedTrend) UnmarshalCBOR(r io.Reader) (err error) { - *t = FeedTrend{} +func (t *FeedPost_Entity) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedPost_Entity{} cr := cbg.NewCborReader(r) @@ -559,37 +944,40 @@ func (t *FeedTrend) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("FeedTrend: map struct too large (%d)", extra) + return fmt.Errorf("FeedPost_Entity: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.LexiconTypeID (string) (string) - case "$type": + switch string(nameBuf[:nameLen]) { + // t.Type (string) (string) + case "type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - t.LexiconTypeID = string(sval) + t.Type = string(sval) } - // t.Subject (schemagen.RepoStrongRef) (struct) - case "subject": + // t.Index (bsky.FeedPost_TextSlice) (struct) + case "index": { @@ -601,34 +989,36 @@ func (t *FeedTrend) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Subject = new(schemagen.RepoStrongRef) - if err := t.Subject.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Subject pointer: %w", err) + t.Index = new(FeedPost_TextSlice) + if err := t.Index.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Index pointer: %w", err) } } } - // t.CreatedAt (string) (string) - case "createdAt": + // t.Value (string) (string) + case "value": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - t.CreatedAt = string(sval) + t.Value = string(sval) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *FeedVote) MarshalCBOR(w io.Writer) error { +func (t *FeedPost_ReplyRef) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -636,95 +1026,46 @@ func (t *FeedVote) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{164}); err != nil { - return err - } - - // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"$type\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("$type")); err != nil { - return err - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.vote"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("app.bsky.feed.vote")); err != nil { - return err - } - - // t.Subject (schemagen.RepoStrongRef) (struct) - if len("subject") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"subject\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("subject")); err != nil { - return err - } - - if err := t.Subject.MarshalCBOR(cw); err != nil { + if _, err := cw.Write([]byte{162}); err != nil { return err } - // t.CreatedAt (string) (string) - if len("createdAt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"createdAt\" was too long") + // t.Root (atproto.RepoStrongRef) (struct) + if len("root") > 1000000 { + return xerrors.Errorf("Value in field \"root\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("root"))); err != nil { return err } - if _, err := io.WriteString(w, string("createdAt")); err != nil { + if _, err := cw.WriteString(string("root")); err != nil { return err } - if len(t.CreatedAt) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.CreatedAt was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.CreatedAt)); err != nil { + if err := t.Root.MarshalCBOR(cw); err != nil { return err } - // t.Direction (string) (string) - if len("direction") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"direction\" was too long") + // t.Parent (atproto.RepoStrongRef) (struct) + if len("parent") > 1000000 { + return xerrors.Errorf("Value in field \"parent\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("direction"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("parent"))); err != nil { return err } - if _, err := io.WriteString(w, string("direction")); err != nil { + if _, err := cw.WriteString(string("parent")); err != nil { return err } - if len(t.Direction) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Direction was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Direction))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Direction)); err != nil { + if err := t.Parent.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *FeedVote) UnmarshalCBOR(r io.Reader) (err error) { - *t = FeedVote{} +func (t *FeedPost_ReplyRef) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedPost_ReplyRef{} cr := cbg.NewCborReader(r) @@ -743,37 +1084,29 @@ func (t *FeedVote) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("FeedVote: map struct too large (%d)", extra) + return fmt.Errorf("FeedPost_ReplyRef: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.LexiconTypeID (string) (string) - case "$type": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.LexiconTypeID = string(sval) - } - // t.Subject (schemagen.RepoStrongRef) (struct) - case "subject": + switch string(nameBuf[:nameLen]) { + // t.Root (atproto.RepoStrongRef) (struct) + case "root": { @@ -785,45 +1118,45 @@ func (t *FeedVote) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Subject = new(schemagen.RepoStrongRef) - if err := t.Subject.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Subject pointer: %w", err) + t.Root = new(atproto.RepoStrongRef) + if err := t.Root.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Root pointer: %w", err) } } } - // t.CreatedAt (string) (string) - case "createdAt": + // t.Parent (atproto.RepoStrongRef) (struct) + case "parent": { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.CreatedAt = string(sval) - } - // t.Direction (string) (string) - case "direction": - { - sval, err := cbg.ReadString(cr) + b, err := cr.ReadByte() if err != nil { return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Parent = new(atproto.RepoStrongRef) + if err := t.Parent.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Parent pointer: %w", err) + } + } - t.Direction = string(sval) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *FeedPost_Entity) MarshalCBOR(w io.Writer) error { +func (t *FeedPost_TextSlice) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -831,99 +1164,59 @@ func (t *FeedPost_Entity) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{164}); err != nil { - return err - } - - // t.Type (string) (string) - if len("type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"type\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("type"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("type")); err != nil { - return err - } - - if len(t.Type) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Type was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Type))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Type)); err != nil { - return err - } - - // t.Index (schemagen.FeedPost_TextSlice) (struct) - if len("index") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"index\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("index"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("index")); err != nil { - return err - } - - if err := t.Index.MarshalCBOR(cw); err != nil { + if _, err := cw.Write([]byte{162}); err != nil { return err } - // t.Value (string) (string) - if len("value") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"value\" was too long") + // t.End (int64) (int64) + if len("end") > 1000000 { + return xerrors.Errorf("Value in field \"end\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("value"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("end"))); err != nil { return err } - if _, err := io.WriteString(w, string("value")); err != nil { + if _, err := cw.WriteString(string("end")); err != nil { return err } - if len(t.Value) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Value was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Value)); err != nil { - return err + if t.End >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.End)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.End-1)); err != nil { + return err + } } - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") + // t.Start (int64) (int64) + if len("start") > 1000000 { + return xerrors.Errorf("Value in field \"start\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("start"))); err != nil { return err } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { + if _, err := cw.WriteString(string("start")); err != nil { return err } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") + if t.Start >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Start)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Start-1)); err != nil { + return err + } } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { - return err - } return nil } -func (t *FeedPost_Entity) UnmarshalCBOR(r io.Reader) (err error) { - *t = FeedPost_Entity{} +func (t *FeedPost_TextSlice) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedPost_TextSlice{} cr := cbg.NewCborReader(r) @@ -942,87 +1235,91 @@ func (t *FeedPost_Entity) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("FeedPost_Entity: map struct too large (%d)", extra) + return fmt.Errorf("FeedPost_TextSlice: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.Type (string) (string) - case "type": - + switch string(nameBuf[:nameLen]) { + // t.End (int64) (int64) + case "end": { - sval, err := cbg.ReadString(cr) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } - t.Type = string(sval) + t.End = int64(extraI) } - // t.Index (schemagen.FeedPost_TextSlice) (struct) - case "index": - + // t.Start (int64) (int64) + case "start": { - - b, err := cr.ReadByte() + maj, extra, err := cr.ReadHeader() if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") } - t.Index = new(FeedPost_TextSlice) - if err := t.Index.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Index pointer: %w", err) + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) } - } - // t.Value (string) (string) - case "value": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Value = string(sval) - } - // t.LexiconTypeID (string) (string) - case "LexiconTypeID": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.LexiconTypeID = string(sval) + t.Start = int64(extraI) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *FeedPost_ReplyRef) MarshalCBOR(w io.Writer) error { +func (t *EmbedImages) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -1030,69 +1327,59 @@ func (t *FeedPost_ReplyRef) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{163}); err != nil { + if _, err := cw.Write([]byte{162}); err != nil { return err } - // t.Root (schemagen.RepoStrongRef) (struct) - if len("root") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"root\" was too long") + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("root"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("root")); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - - if err := t.Root.MarshalCBOR(cw); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } - // t.Parent (schemagen.RepoStrongRef) (struct) - if len("parent") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"parent\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("parent"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("parent")); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.embed.images"))); err != nil { return err } - - if err := t.Parent.MarshalCBOR(cw); err != nil { + if _, err := cw.WriteString(string("app.bsky.embed.images")); err != nil { return err } - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") + // t.Images ([]*bsky.EmbedImages_Image) (slice) + if len("images") > 1000000 { + return xerrors.Errorf("Value in field \"images\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("images"))); err != nil { return err } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { + if _, err := cw.WriteString(string("images")); err != nil { return err } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") + if len(t.Images) > 8192 { + return xerrors.Errorf("Slice value in field t.Images was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Images))); err != nil { return err } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { - return err + for _, v := range t.Images { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + } return nil } -func (t *FeedPost_ReplyRef) UnmarshalCBOR(r io.Reader) (err error) { - *t = FeedPost_ReplyRef{} +func (t *EmbedImages) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedImages{} cr := cbg.NewCborReader(r) @@ -1111,85 +1398,99 @@ func (t *FeedPost_ReplyRef) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("FeedPost_ReplyRef: map struct too large (%d)", extra) + return fmt.Errorf("EmbedImages: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.Root (schemagen.RepoStrongRef) (struct) - case "root": + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": { - - b, err := cr.ReadByte() + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Root = new(schemagen.RepoStrongRef) - if err := t.Root.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Root pointer: %w", err) - } - } + t.LexiconTypeID = string(sval) } - // t.Parent (schemagen.RepoStrongRef) (struct) - case "parent": + // t.Images ([]*bsky.EmbedImages_Image) (slice) + case "images": - { + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Parent = new(schemagen.RepoStrongRef) - if err := t.Parent.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Parent pointer: %w", err) - } - } + if extra > 8192 { + return fmt.Errorf("t.Images: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Images = make([]*EmbedImages_Image, extra) } - // t.LexiconTypeID (string) (string) - case "LexiconTypeID": - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Images[i] = new(EmbedImages_Image) + if err := t.Images[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Images[i] pointer: %w", err) + } + } - t.LexiconTypeID = string(sval) + } + + } } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *FeedPost_TextSlice) MarshalCBOR(w io.Writer) error { +func (t *EmbedExternal) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -1197,81 +1498,49 @@ func (t *FeedPost_TextSlice) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{163}); err != nil { + if _, err := cw.Write([]byte{162}); err != nil { return err } - // t.End (int64) (int64) - if len("end") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"end\" was too long") + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("end"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("end")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } - if t.End >= 0 { - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.End)); err != nil { - return err - } - } else { - if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.End-1)); err != nil { - return err - } - } - - // t.Start (int64) (int64) - if len("start") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"start\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("start"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.embed.external"))); err != nil { return err } - if _, err := io.WriteString(w, string("start")); err != nil { + if _, err := cw.WriteString(string("app.bsky.embed.external")); err != nil { return err } - if t.Start >= 0 { - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Start)); err != nil { - return err - } - } else { - if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Start-1)); err != nil { - return err - } - } - - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") + // t.External (bsky.EmbedExternal_External) (struct) + if len("external") > 1000000 { + return xerrors.Errorf("Value in field \"external\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("external"))); err != nil { return err } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { + if _, err := cw.WriteString(string("external")); err != nil { return err } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { + if err := t.External.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *FeedPost_TextSlice) UnmarshalCBOR(r io.Reader) (err error) { - *t = FeedPost_TextSlice{} +func (t *EmbedExternal) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedExternal{} cr := cbg.NewCborReader(r) @@ -1290,160 +1559,178 @@ func (t *FeedPost_TextSlice) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("FeedPost_TextSlice: map struct too large (%d)", extra) + return fmt.Errorf("EmbedExternal: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 8) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.End (int64) (int64) - case "end": + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + { - maj, extra, err := cr.ReadHeader() - var extraI int64 + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - t.End = int64(extraI) + t.LexiconTypeID = string(sval) } - // t.Start (int64) (int64) - case "start": + // t.External (bsky.EmbedExternal_External) (struct) + case "external": + { - maj, extra, err := cr.ReadHeader() - var extraI int64 + + b, err := cr.ReadByte() if err != nil { return err } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") + t.External = new(EmbedExternal_External) + if err := t.External.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.External pointer: %w", err) } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.Start = int64(extraI) - } - // t.LexiconTypeID (string) (string) - case "LexiconTypeID": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err } - t.LexiconTypeID = string(sval) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *EmbedImages) MarshalCBOR(w io.Writer) error { +func (t *EmbedExternal_External) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) + fieldCount := 4 - if _, err := cw.Write([]byte{162}); err != nil { + if t.Thumb == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } - // t.Images ([]*schemagen.EmbedImages_Image) (slice) - if len("images") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"images\" was too long") + // t.Uri (string) (string) + if len("uri") > 1000000 { + return xerrors.Errorf("Value in field \"uri\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("images"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { return err } - if _, err := io.WriteString(w, string("images")); err != nil { + if _, err := cw.WriteString(string("uri")); err != nil { return err } - if len(t.Images) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field t.Images was too long") + if len(t.Uri) > 1000000 { + return xerrors.Errorf("Value in field t.Uri was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Images))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { return err } - for _, v := range t.Images { - if err := v.MarshalCBOR(cw); err != nil { + if _, err := cw.WriteString(string(t.Uri)); err != nil { + return err + } + + // t.Thumb (util.LexBlob) (struct) + if t.Thumb != nil { + + if len("thumb") > 1000000 { + return xerrors.Errorf("Value in field \"thumb\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("thumb"))); err != nil { + return err + } + if _, err := cw.WriteString(string("thumb")); err != nil { + return err + } + + if err := t.Thumb.MarshalCBOR(cw); err != nil { return err } } - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") + // t.Title (string) (string) + if len("title") > 1000000 { + return xerrors.Errorf("Value in field \"title\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("title"))); err != nil { + return err + } + if _, err := cw.WriteString(string("title")); err != nil { + return err + } + + if len(t.Title) > 1000000 { + return xerrors.Errorf("Value in field t.Title was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Title))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Title)); err != nil { + return err + } + + // t.Description (string) (string) + if len("description") > 1000000 { + return xerrors.Errorf("Value in field \"description\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("description"))); err != nil { return err } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { + if _, err := cw.WriteString(string("description")); err != nil { return err } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") + if len(t.Description) > 1000000 { + return xerrors.Errorf("Value in field t.Description was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Description))); err != nil { return err } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { + if _, err := cw.WriteString(string(t.Description)); err != nil { return err } return nil } -func (t *EmbedImages) UnmarshalCBOR(r io.Reader) (err error) { - *t = EmbedImages{} +func (t *EmbedExternal_External) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedExternal_External{} cr := cbg.NewCborReader(r) @@ -1462,182 +1749,170 @@ func (t *EmbedImages) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("EmbedImages: map struct too large (%d)", extra) + return fmt.Errorf("EmbedExternal_External: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.Images ([]*schemagen.EmbedImages_Image) (slice) - case "images": + switch string(nameBuf[:nameLen]) { + // t.Uri (string) (string) + case "uri": - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } - if extra > cbg.MaxLength { - return fmt.Errorf("t.Images: array too large (%d)", extra) + t.Uri = string(sval) } + // t.Thumb (util.LexBlob) (struct) + case "thumb": - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } + { - if extra > 0 { - t.Images = make([]*EmbedImages_Image, extra) - } + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Thumb = new(util.LexBlob) + if err := t.Thumb.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Thumb pointer: %w", err) + } + } - for i := 0; i < int(extra); i++ { + } + // t.Title (string) (string) + case "title": - var v EmbedImages_Image - if err := v.UnmarshalCBOR(cr); err != nil { + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { return err } - t.Images[i] = &v + t.Title = string(sval) } - - // t.LexiconTypeID (string) (string) - case "LexiconTypeID": + // t.Description (string) (string) + case "description": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - t.LexiconTypeID = string(sval) + t.Description = string(sval) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *EmbedImages_PresentedImage) MarshalCBOR(w io.Writer) error { +func (t *EmbedImages_Image) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) + fieldCount := 3 - if _, err := cw.Write([]byte{164}); err != nil { + if t.AspectRatio == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } // t.Alt (string) (string) - if len("alt") > cbg.MaxLength { + if len("alt") > 1000000 { return xerrors.Errorf("Value in field \"alt\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("alt"))); err != nil { return err } - if _, err := io.WriteString(w, string("alt")); err != nil { + if _, err := cw.WriteString(string("alt")); err != nil { return err } - if len(t.Alt) > cbg.MaxLength { + if len(t.Alt) > 1000000 { return xerrors.Errorf("Value in field t.Alt was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Alt))); err != nil { return err } - if _, err := io.WriteString(w, string(t.Alt)); err != nil { - return err - } - - // t.Thumb (string) (string) - if len("thumb") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"thumb\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("thumb"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("thumb")); err != nil { - return err - } - - if len(t.Thumb) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Thumb was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Thumb))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Thumb)); err != nil { + if _, err := cw.WriteString(string(t.Alt)); err != nil { return err } - // t.Fullsize (string) (string) - if len("fullsize") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"fullsize\" was too long") + // t.Image (util.LexBlob) (struct) + if len("image") > 1000000 { + return xerrors.Errorf("Value in field \"image\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fullsize"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("image"))); err != nil { return err } - if _, err := io.WriteString(w, string("fullsize")); err != nil { + if _, err := cw.WriteString(string("image")); err != nil { return err } - if len(t.Fullsize) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Fullsize was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Fullsize))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Fullsize)); err != nil { + if err := t.Image.MarshalCBOR(cw); err != nil { return err } - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") - } + // t.AspectRatio (bsky.EmbedDefs_AspectRatio) (struct) + if t.AspectRatio != nil { - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { - return err - } + if len("aspectRatio") > 1000000 { + return xerrors.Errorf("Value in field \"aspectRatio\" was too long") + } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("aspectRatio"))); err != nil { + return err + } + if _, err := cw.WriteString(string("aspectRatio")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { - return err + if err := t.AspectRatio.MarshalCBOR(cw); err != nil { + return err + } } return nil } -func (t *EmbedImages_PresentedImage) UnmarshalCBOR(r io.Reader) (err error) { - *t = EmbedImages_PresentedImage{} +func (t *EmbedImages_Image) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedImages_Image{} cr := cbg.NewCborReader(r) @@ -1656,132 +1931,194 @@ func (t *EmbedImages_PresentedImage) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("EmbedImages_PresentedImage: map struct too large (%d)", extra) + return fmt.Errorf("EmbedImages_Image: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.Alt (string) (string) case "alt": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.Alt = string(sval) } - // t.Thumb (string) (string) - case "thumb": + // t.Image (util.LexBlob) (struct) + case "image": { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - t.Thumb = string(sval) - } - // t.Fullsize (string) (string) - case "fullsize": - - { - sval, err := cbg.ReadString(cr) + b, err := cr.ReadByte() if err != nil { return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Image = new(util.LexBlob) + if err := t.Image.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Image pointer: %w", err) + } + } - t.Fullsize = string(sval) } - // t.LexiconTypeID (string) (string) - case "LexiconTypeID": + // t.AspectRatio (bsky.EmbedDefs_AspectRatio) (struct) + case "aspectRatio": { - sval, err := cbg.ReadString(cr) + + b, err := cr.ReadByte() if err != nil { return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.AspectRatio = new(EmbedDefs_AspectRatio) + if err := t.AspectRatio.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.AspectRatio pointer: %w", err) + } + } - t.LexiconTypeID = string(sval) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *EmbedExternal) MarshalCBOR(w io.Writer) error { +func (t *GraphFollow) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) + fieldCount := 4 - if _, err := cw.Write([]byte{162}); err != nil { + if t.Via == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } - // t.External (schemagen.EmbedExternal_External) (struct) - if len("external") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"external\" was too long") + // t.Via (atproto.RepoStrongRef) (struct) + if t.Via != nil { + + if len("via") > 1000000 { + return xerrors.Errorf("Value in field \"via\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("via"))); err != nil { + return err + } + if _, err := cw.WriteString(string("via")); err != nil { + return err + } + + if err := t.Via.MarshalCBOR(cw); err != nil { + return err + } } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("external"))); err != nil { + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("external")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } - if err := t.External.MarshalCBOR(cw); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.follow"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.graph.follow")); err != nil { return err } - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") + // t.Subject (string) (string) + if len("subject") > 1000000 { + return xerrors.Errorf("Value in field \"subject\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + return err + } + if _, err := cw.WriteString(string("subject")); err != nil { + return err + } + + if len(t.Subject) > 1000000 { + return xerrors.Errorf("Value in field t.Subject was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Subject))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Subject)); err != nil { + return err + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { return err } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { + if _, err := cw.WriteString(string("createdAt")); err != nil { return err } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { return err } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } return nil } -func (t *EmbedExternal) UnmarshalCBOR(r io.Reader) (err error) { - *t = EmbedExternal{} +func (t *GraphFollow) UnmarshalCBOR(r io.Reader) (err error) { + *t = GraphFollow{} cr := cbg.NewCborReader(r) @@ -1800,26 +2137,29 @@ func (t *EmbedExternal) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("EmbedExternal: map struct too large (%d)", extra) + return fmt.Errorf("GraphFollow: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.External (schemagen.EmbedExternal_External) (struct) - case "external": + switch string(nameBuf[:nameLen]) { + // t.Via (atproto.RepoStrongRef) (struct) + case "via": { @@ -1831,157 +2171,5225 @@ func (t *EmbedExternal) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.External = new(EmbedExternal_External) - if err := t.External.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.External pointer: %w", err) + t.Via = new(atproto.RepoStrongRef) + if err := t.Via.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Via pointer: %w", err) } } } // t.LexiconTypeID (string) (string) - case "LexiconTypeID": + case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.LexiconTypeID = string(sval) } + // t.Subject (string) (string) + case "subject": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Subject = string(sval) + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *ActorProfile) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 11 + + if t.Avatar == nil { + fieldCount-- + } + + if t.Banner == nil { + fieldCount-- + } + + if t.CreatedAt == nil { + fieldCount-- + } + + if t.Description == nil { + fieldCount-- + } + + if t.DisplayName == nil { + fieldCount-- + } + + if t.JoinedViaStarterPack == nil { + fieldCount-- + } + + if t.Labels == nil { + fieldCount-- + } + + if t.PinnedPost == nil { + fieldCount-- + } + + if t.Pronouns == nil { + fieldCount-- + } + + if t.Website == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.actor.profile"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.actor.profile")); err != nil { + return err + } + + // t.Avatar (util.LexBlob) (struct) + if t.Avatar != nil { + + if len("avatar") > 1000000 { + return xerrors.Errorf("Value in field \"avatar\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("avatar"))); err != nil { + return err + } + if _, err := cw.WriteString(string("avatar")); err != nil { + return err + } + + if err := t.Avatar.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Banner (util.LexBlob) (struct) + if t.Banner != nil { + + if len("banner") > 1000000 { + return xerrors.Errorf("Value in field \"banner\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("banner"))); err != nil { + return err + } + if _, err := cw.WriteString(string("banner")); err != nil { + return err + } + + if err := t.Banner.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Labels (bsky.ActorProfile_Labels) (struct) + if t.Labels != nil { + + if len("labels") > 1000000 { + return xerrors.Errorf("Value in field \"labels\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("labels"))); err != nil { + return err + } + if _, err := cw.WriteString(string("labels")); err != nil { + return err + } + + if err := t.Labels.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Website (string) (string) + if t.Website != nil { + + if len("website") > 1000000 { + return xerrors.Errorf("Value in field \"website\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("website"))); err != nil { + return err + } + if _, err := cw.WriteString(string("website")); err != nil { + return err + } + + if t.Website == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Website) > 1000000 { + return xerrors.Errorf("Value in field t.Website was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Website))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Website)); err != nil { + return err + } + } + } + + // t.Pronouns (string) (string) + if t.Pronouns != nil { + + if len("pronouns") > 1000000 { + return xerrors.Errorf("Value in field \"pronouns\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pronouns"))); err != nil { + return err + } + if _, err := cw.WriteString(string("pronouns")); err != nil { + return err + } + + if t.Pronouns == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Pronouns) > 1000000 { + return xerrors.Errorf("Value in field t.Pronouns was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Pronouns))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Pronouns)); err != nil { + return err + } + } + } + + // t.CreatedAt (string) (string) + if t.CreatedAt != nil { + + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if t.CreatedAt == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.CreatedAt)); err != nil { + return err + } + } + } + + // t.PinnedPost (atproto.RepoStrongRef) (struct) + if t.PinnedPost != nil { + + if len("pinnedPost") > 1000000 { + return xerrors.Errorf("Value in field \"pinnedPost\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pinnedPost"))); err != nil { + return err + } + if _, err := cw.WriteString(string("pinnedPost")); err != nil { + return err + } + + if err := t.PinnedPost.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Description (string) (string) + if t.Description != nil { + + if len("description") > 1000000 { + return xerrors.Errorf("Value in field \"description\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("description"))); err != nil { + return err + } + if _, err := cw.WriteString(string("description")); err != nil { + return err + } + + if t.Description == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Description) > 1000000 { + return xerrors.Errorf("Value in field t.Description was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Description))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Description)); err != nil { + return err + } + } + } + + // t.DisplayName (string) (string) + if t.DisplayName != nil { + + if len("displayName") > 1000000 { + return xerrors.Errorf("Value in field \"displayName\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("displayName"))); err != nil { + return err + } + if _, err := cw.WriteString(string("displayName")); err != nil { + return err + } + + if t.DisplayName == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.DisplayName) > 1000000 { + return xerrors.Errorf("Value in field t.DisplayName was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.DisplayName))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.DisplayName)); err != nil { + return err + } + } + } + + // t.JoinedViaStarterPack (atproto.RepoStrongRef) (struct) + if t.JoinedViaStarterPack != nil { + + if len("joinedViaStarterPack") > 1000000 { + return xerrors.Errorf("Value in field \"joinedViaStarterPack\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("joinedViaStarterPack"))); err != nil { + return err + } + if _, err := cw.WriteString(string("joinedViaStarterPack")); err != nil { + return err + } + + if err := t.JoinedViaStarterPack.MarshalCBOR(cw); err != nil { + return err + } + } + return nil +} + +func (t *ActorProfile) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorProfile{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorProfile: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 20) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Avatar (util.LexBlob) (struct) + case "avatar": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Avatar = new(util.LexBlob) + if err := t.Avatar.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Avatar pointer: %w", err) + } + } + + } + // t.Banner (util.LexBlob) (struct) + case "banner": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Banner = new(util.LexBlob) + if err := t.Banner.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Banner pointer: %w", err) + } + } + + } + // t.Labels (bsky.ActorProfile_Labels) (struct) + case "labels": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Labels = new(ActorProfile_Labels) + if err := t.Labels.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Labels pointer: %w", err) + } + } + + } + // t.Website (string) (string) + case "website": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Website = (*string)(&sval) + } + } + // t.Pronouns (string) (string) + case "pronouns": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Pronouns = (*string)(&sval) + } + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = (*string)(&sval) + } + } + // t.PinnedPost (atproto.RepoStrongRef) (struct) + case "pinnedPost": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.PinnedPost = new(atproto.RepoStrongRef) + if err := t.PinnedPost.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.PinnedPost pointer: %w", err) + } + } + + } + // t.Description (string) (string) + case "description": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Description = (*string)(&sval) + } + } + // t.DisplayName (string) (string) + case "displayName": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.DisplayName = (*string)(&sval) + } + } + // t.JoinedViaStarterPack (atproto.RepoStrongRef) (struct) + case "joinedViaStarterPack": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.JoinedViaStarterPack = new(atproto.RepoStrongRef) + if err := t.JoinedViaStarterPack.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.JoinedViaStarterPack pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *EmbedRecord) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.embed.record"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.embed.record")); err != nil { + return err + } + + // t.Record (atproto.RepoStrongRef) (struct) + if len("record") > 1000000 { + return xerrors.Errorf("Value in field \"record\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("record"))); err != nil { + return err + } + if _, err := cw.WriteString(string("record")); err != nil { + return err + } + + if err := t.Record.MarshalCBOR(cw); err != nil { + return err + } + return nil +} + +func (t *EmbedRecord) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedRecord{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("EmbedRecord: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Record (atproto.RepoStrongRef) (struct) + case "record": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Record = new(atproto.RepoStrongRef) + if err := t.Record.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Record pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedLike) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 4 + + if t.Via == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Via (atproto.RepoStrongRef) (struct) + if t.Via != nil { + + if len("via") > 1000000 { + return xerrors.Errorf("Value in field \"via\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("via"))); err != nil { + return err + } + if _, err := cw.WriteString(string("via")); err != nil { + return err + } + + if err := t.Via.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.like"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.feed.like")); err != nil { + return err + } + + // t.Subject (atproto.RepoStrongRef) (struct) + if len("subject") > 1000000 { + return xerrors.Errorf("Value in field \"subject\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + return err + } + if _, err := cw.WriteString(string("subject")); err != nil { + return err + } + + if err := t.Subject.MarshalCBOR(cw); err != nil { + return err + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + return nil +} + +func (t *FeedLike) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedLike{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedLike: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 9) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Via (atproto.RepoStrongRef) (struct) + case "via": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Via = new(atproto.RepoStrongRef) + if err := t.Via.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Via pointer: %w", err) + } + } + + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Subject (atproto.RepoStrongRef) (struct) + case "subject": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Subject = new(atproto.RepoStrongRef) + if err := t.Subject.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Subject pointer: %w", err) + } + } + + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *RichtextFacet) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Index (bsky.RichtextFacet_ByteSlice) (struct) + if len("index") > 1000000 { + return xerrors.Errorf("Value in field \"index\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("index"))); err != nil { + return err + } + if _, err := cw.WriteString(string("index")); err != nil { + return err + } + + if err := t.Index.MarshalCBOR(cw); err != nil { + return err + } + + // t.Features ([]*bsky.RichtextFacet_Features_Elem) (slice) + if len("features") > 1000000 { + return xerrors.Errorf("Value in field \"features\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("features"))); err != nil { + return err + } + if _, err := cw.WriteString(string("features")); err != nil { + return err + } + + if len(t.Features) > 8192 { + return xerrors.Errorf("Slice value in field t.Features was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Features))); err != nil { + return err + } + for _, v := range t.Features { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + return nil +} + +func (t *RichtextFacet) UnmarshalCBOR(r io.Reader) (err error) { + *t = RichtextFacet{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("RichtextFacet: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 8) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Index (bsky.RichtextFacet_ByteSlice) (struct) + case "index": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Index = new(RichtextFacet_ByteSlice) + if err := t.Index.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Index pointer: %w", err) + } + } + + } + // t.Features ([]*bsky.RichtextFacet_Features_Elem) (slice) + case "features": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Features: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Features = make([]*RichtextFacet_Features_Elem, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Features[i] = new(RichtextFacet_Features_Elem) + if err := t.Features[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Features[i] pointer: %w", err) + } + } + + } + + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *RichtextFacet_ByteSlice) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.ByteEnd (int64) (int64) + if len("byteEnd") > 1000000 { + return xerrors.Errorf("Value in field \"byteEnd\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("byteEnd"))); err != nil { + return err + } + if _, err := cw.WriteString(string("byteEnd")); err != nil { + return err + } + + if t.ByteEnd >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ByteEnd)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.ByteEnd-1)); err != nil { + return err + } + } + + // t.ByteStart (int64) (int64) + if len("byteStart") > 1000000 { + return xerrors.Errorf("Value in field \"byteStart\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("byteStart"))); err != nil { + return err + } + if _, err := cw.WriteString(string("byteStart")); err != nil { + return err + } + + if t.ByteStart >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ByteStart)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.ByteStart-1)); err != nil { + return err + } + } + + return nil +} + +func (t *RichtextFacet_ByteSlice) UnmarshalCBOR(r io.Reader) (err error) { + *t = RichtextFacet_ByteSlice{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("RichtextFacet_ByteSlice: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 9) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.ByteEnd (int64) (int64) + case "byteEnd": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.ByteEnd = int64(extraI) + } + // t.ByteStart (int64) (int64) + case "byteStart": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.ByteStart = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *RichtextFacet_Link) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Uri (string) (string) + if len("uri") > 1000000 { + return xerrors.Errorf("Value in field \"uri\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { + return err + } + if _, err := cw.WriteString(string("uri")); err != nil { + return err + } + + if len(t.Uri) > 1000000 { + return xerrors.Errorf("Value in field t.Uri was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Uri)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.richtext.facet#link"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.richtext.facet#link")); err != nil { + return err + } + return nil +} + +func (t *RichtextFacet_Link) UnmarshalCBOR(r io.Reader) (err error) { + *t = RichtextFacet_Link{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("RichtextFacet_Link: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Uri (string) (string) + case "uri": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Uri = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *RichtextFacet_Mention) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Did (string) (string) + if len("did") > 1000000 { + return xerrors.Errorf("Value in field \"did\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil { + return err + } + if _, err := cw.WriteString(string("did")); err != nil { + return err + } + + if len(t.Did) > 1000000 { + return xerrors.Errorf("Value in field t.Did was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Did))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Did)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.richtext.facet#mention"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.richtext.facet#mention")); err != nil { + return err + } + return nil +} + +func (t *RichtextFacet_Mention) UnmarshalCBOR(r io.Reader) (err error) { + *t = RichtextFacet_Mention{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("RichtextFacet_Mention: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Did (string) (string) + case "did": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Did = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *RichtextFacet_Tag) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Tag (string) (string) + if len("tag") > 1000000 { + return xerrors.Errorf("Value in field \"tag\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("tag"))); err != nil { + return err + } + if _, err := cw.WriteString(string("tag")); err != nil { + return err + } + + if len(t.Tag) > 1000000 { + return xerrors.Errorf("Value in field t.Tag was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Tag))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Tag)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.richtext.facet#tag"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.richtext.facet#tag")); err != nil { + return err + } + return nil +} + +func (t *RichtextFacet_Tag) UnmarshalCBOR(r io.Reader) (err error) { + *t = RichtextFacet_Tag{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("RichtextFacet_Tag: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Tag (string) (string) + case "tag": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Tag = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *EmbedRecordWithMedia) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.embed.recordWithMedia"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.embed.recordWithMedia")); err != nil { + return err + } + + // t.Media (bsky.EmbedRecordWithMedia_Media) (struct) + if len("media") > 1000000 { + return xerrors.Errorf("Value in field \"media\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("media"))); err != nil { + return err + } + if _, err := cw.WriteString(string("media")); err != nil { + return err + } + + if err := t.Media.MarshalCBOR(cw); err != nil { + return err + } + + // t.Record (bsky.EmbedRecord) (struct) + if len("record") > 1000000 { + return xerrors.Errorf("Value in field \"record\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("record"))); err != nil { + return err + } + if _, err := cw.WriteString(string("record")); err != nil { + return err + } + + if err := t.Record.MarshalCBOR(cw); err != nil { + return err + } + return nil +} + +func (t *EmbedRecordWithMedia) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedRecordWithMedia{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("EmbedRecordWithMedia: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Media (bsky.EmbedRecordWithMedia_Media) (struct) + case "media": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Media = new(EmbedRecordWithMedia_Media) + if err := t.Media.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Media pointer: %w", err) + } + } + + } + // t.Record (bsky.EmbedRecord) (struct) + case "record": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Record = new(EmbedRecord) + if err := t.Record.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Record pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedDefs_NotFoundPost) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Uri (string) (string) + if len("uri") > 1000000 { + return xerrors.Errorf("Value in field \"uri\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { + return err + } + if _, err := cw.WriteString(string("uri")); err != nil { + return err + } + + if len(t.Uri) > 1000000 { + return xerrors.Errorf("Value in field t.Uri was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Uri)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.defs#notFoundPost"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.feed.defs#notFoundPost")); err != nil { + return err + } + + // t.NotFound (bool) (bool) + if len("notFound") > 1000000 { + return xerrors.Errorf("Value in field \"notFound\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("notFound"))); err != nil { + return err + } + if _, err := cw.WriteString(string("notFound")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.NotFound); err != nil { + return err + } + return nil +} + +func (t *FeedDefs_NotFoundPost) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedDefs_NotFoundPost{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedDefs_NotFoundPost: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 8) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Uri (string) (string) + case "uri": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Uri = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.NotFound (bool) (bool) + case "notFound": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.NotFound = false + case 21: + t.NotFound = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *GraphBlock) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.block"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.graph.block")); err != nil { + return err + } + + // t.Subject (string) (string) + if len("subject") > 1000000 { + return xerrors.Errorf("Value in field \"subject\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + return err + } + if _, err := cw.WriteString(string("subject")); err != nil { + return err + } + + if len(t.Subject) > 1000000 { + return xerrors.Errorf("Value in field t.Subject was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Subject))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Subject)); err != nil { + return err + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + return nil +} + +func (t *GraphBlock) UnmarshalCBOR(r io.Reader) (err error) { + *t = GraphBlock{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("GraphBlock: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 9) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Subject (string) (string) + case "subject": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Subject = string(sval) + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *GraphList) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 8 + + if t.Avatar == nil { + fieldCount-- + } + + if t.Description == nil { + fieldCount-- + } + + if t.DescriptionFacets == nil { + fieldCount-- + } + + if t.Labels == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Name (string) (string) + if len("name") > 1000000 { + return xerrors.Errorf("Value in field \"name\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := cw.WriteString(string("name")); err != nil { + return err + } + + if len(t.Name) > 1000000 { + return xerrors.Errorf("Value in field t.Name was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Name)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.list"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.graph.list")); err != nil { + return err + } + + // t.Avatar (util.LexBlob) (struct) + if t.Avatar != nil { + + if len("avatar") > 1000000 { + return xerrors.Errorf("Value in field \"avatar\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("avatar"))); err != nil { + return err + } + if _, err := cw.WriteString(string("avatar")); err != nil { + return err + } + + if err := t.Avatar.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Labels (bsky.GraphList_Labels) (struct) + if t.Labels != nil { + + if len("labels") > 1000000 { + return xerrors.Errorf("Value in field \"labels\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("labels"))); err != nil { + return err + } + if _, err := cw.WriteString(string("labels")); err != nil { + return err + } + + if err := t.Labels.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Purpose (string) (string) + if len("purpose") > 1000000 { + return xerrors.Errorf("Value in field \"purpose\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("purpose"))); err != nil { + return err + } + if _, err := cw.WriteString(string("purpose")); err != nil { + return err + } + + if t.Purpose == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Purpose) > 1000000 { + return xerrors.Errorf("Value in field t.Purpose was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Purpose))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Purpose)); err != nil { + return err + } + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + + // t.Description (string) (string) + if t.Description != nil { + + if len("description") > 1000000 { + return xerrors.Errorf("Value in field \"description\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("description"))); err != nil { + return err + } + if _, err := cw.WriteString(string("description")); err != nil { + return err + } + + if t.Description == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Description) > 1000000 { + return xerrors.Errorf("Value in field t.Description was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Description))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Description)); err != nil { + return err + } + } + } + + // t.DescriptionFacets ([]*bsky.RichtextFacet) (slice) + if t.DescriptionFacets != nil { + + if len("descriptionFacets") > 1000000 { + return xerrors.Errorf("Value in field \"descriptionFacets\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("descriptionFacets"))); err != nil { + return err + } + if _, err := cw.WriteString(string("descriptionFacets")); err != nil { + return err + } + + if len(t.DescriptionFacets) > 8192 { + return xerrors.Errorf("Slice value in field t.DescriptionFacets was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.DescriptionFacets))); err != nil { + return err + } + for _, v := range t.DescriptionFacets { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + } + return nil +} + +func (t *GraphList) UnmarshalCBOR(r io.Reader) (err error) { + *t = GraphList{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("GraphList: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 17) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Name (string) (string) + case "name": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Name = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Avatar (util.LexBlob) (struct) + case "avatar": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Avatar = new(util.LexBlob) + if err := t.Avatar.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Avatar pointer: %w", err) + } + } + + } + // t.Labels (bsky.GraphList_Labels) (struct) + case "labels": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Labels = new(GraphList_Labels) + if err := t.Labels.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Labels pointer: %w", err) + } + } + + } + // t.Purpose (string) (string) + case "purpose": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Purpose = (*string)(&sval) + } + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + // t.Description (string) (string) + case "description": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Description = (*string)(&sval) + } + } + // t.DescriptionFacets ([]*bsky.RichtextFacet) (slice) + case "descriptionFacets": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.DescriptionFacets: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.DescriptionFacets = make([]*RichtextFacet, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.DescriptionFacets[i] = new(RichtextFacet) + if err := t.DescriptionFacets[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.DescriptionFacets[i] pointer: %w", err) + } + } + + } + + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *GraphListitem) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.List (string) (string) + if len("list") > 1000000 { + return xerrors.Errorf("Value in field \"list\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("list"))); err != nil { + return err + } + if _, err := cw.WriteString(string("list")); err != nil { + return err + } + + if len(t.List) > 1000000 { + return xerrors.Errorf("Value in field t.List was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.List))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.List)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.listitem"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.graph.listitem")); err != nil { + return err + } + + // t.Subject (string) (string) + if len("subject") > 1000000 { + return xerrors.Errorf("Value in field \"subject\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + return err + } + if _, err := cw.WriteString(string("subject")); err != nil { + return err + } + + if len(t.Subject) > 1000000 { + return xerrors.Errorf("Value in field t.Subject was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Subject))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Subject)); err != nil { + return err + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + return nil +} + +func (t *GraphListitem) UnmarshalCBOR(r io.Reader) (err error) { + *t = GraphListitem{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("GraphListitem: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 9) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.List (string) (string) + case "list": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.List = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Subject (string) (string) + case "subject": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Subject = string(sval) + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedGenerator) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 10 + + if t.AcceptsInteractions == nil { + fieldCount-- + } + + if t.Avatar == nil { + fieldCount-- + } + + if t.ContentMode == nil { + fieldCount-- + } + + if t.Description == nil { + fieldCount-- + } + + if t.DescriptionFacets == nil { + fieldCount-- + } + + if t.Labels == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Did (string) (string) + if len("did") > 1000000 { + return xerrors.Errorf("Value in field \"did\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil { + return err + } + if _, err := cw.WriteString(string("did")); err != nil { + return err + } + + if len(t.Did) > 1000000 { + return xerrors.Errorf("Value in field t.Did was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Did))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Did)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.generator"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.feed.generator")); err != nil { + return err + } + + // t.Avatar (util.LexBlob) (struct) + if t.Avatar != nil { + + if len("avatar") > 1000000 { + return xerrors.Errorf("Value in field \"avatar\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("avatar"))); err != nil { + return err + } + if _, err := cw.WriteString(string("avatar")); err != nil { + return err + } + + if err := t.Avatar.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Labels (bsky.FeedGenerator_Labels) (struct) + if t.Labels != nil { + + if len("labels") > 1000000 { + return xerrors.Errorf("Value in field \"labels\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("labels"))); err != nil { + return err + } + if _, err := cw.WriteString(string("labels")); err != nil { + return err + } + + if err := t.Labels.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + + // t.ContentMode (string) (string) + if t.ContentMode != nil { + + if len("contentMode") > 1000000 { + return xerrors.Errorf("Value in field \"contentMode\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("contentMode"))); err != nil { + return err + } + if _, err := cw.WriteString(string("contentMode")); err != nil { + return err + } + + if t.ContentMode == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.ContentMode) > 1000000 { + return xerrors.Errorf("Value in field t.ContentMode was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.ContentMode))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.ContentMode)); err != nil { + return err + } + } + } + + // t.Description (string) (string) + if t.Description != nil { + + if len("description") > 1000000 { + return xerrors.Errorf("Value in field \"description\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("description"))); err != nil { + return err + } + if _, err := cw.WriteString(string("description")); err != nil { + return err + } + + if t.Description == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Description) > 1000000 { + return xerrors.Errorf("Value in field t.Description was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Description))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Description)); err != nil { + return err + } + } + } + + // t.DisplayName (string) (string) + if len("displayName") > 1000000 { + return xerrors.Errorf("Value in field \"displayName\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("displayName"))); err != nil { + return err + } + if _, err := cw.WriteString(string("displayName")); err != nil { + return err + } + + if len(t.DisplayName) > 1000000 { + return xerrors.Errorf("Value in field t.DisplayName was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.DisplayName))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.DisplayName)); err != nil { + return err + } + + // t.DescriptionFacets ([]*bsky.RichtextFacet) (slice) + if t.DescriptionFacets != nil { + + if len("descriptionFacets") > 1000000 { + return xerrors.Errorf("Value in field \"descriptionFacets\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("descriptionFacets"))); err != nil { + return err + } + if _, err := cw.WriteString(string("descriptionFacets")); err != nil { + return err + } + + if len(t.DescriptionFacets) > 8192 { + return xerrors.Errorf("Slice value in field t.DescriptionFacets was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.DescriptionFacets))); err != nil { + return err + } + for _, v := range t.DescriptionFacets { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + } + + // t.AcceptsInteractions (bool) (bool) + if t.AcceptsInteractions != nil { + + if len("acceptsInteractions") > 1000000 { + return xerrors.Errorf("Value in field \"acceptsInteractions\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("acceptsInteractions"))); err != nil { + return err + } + if _, err := cw.WriteString(string("acceptsInteractions")); err != nil { + return err + } + + if t.AcceptsInteractions == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteBool(w, *t.AcceptsInteractions); err != nil { + return err + } + } + } + return nil +} + +func (t *FeedGenerator) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedGenerator{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedGenerator: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 19) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Did (string) (string) + case "did": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Did = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Avatar (util.LexBlob) (struct) + case "avatar": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Avatar = new(util.LexBlob) + if err := t.Avatar.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Avatar pointer: %w", err) + } + } + + } + // t.Labels (bsky.FeedGenerator_Labels) (struct) + case "labels": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Labels = new(FeedGenerator_Labels) + if err := t.Labels.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Labels pointer: %w", err) + } + } + + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + // t.ContentMode (string) (string) + case "contentMode": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.ContentMode = (*string)(&sval) + } + } + // t.Description (string) (string) + case "description": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Description = (*string)(&sval) + } + } + // t.DisplayName (string) (string) + case "displayName": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.DisplayName = string(sval) + } + // t.DescriptionFacets ([]*bsky.RichtextFacet) (slice) + case "descriptionFacets": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.DescriptionFacets: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.DescriptionFacets = make([]*RichtextFacet, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.DescriptionFacets[i] = new(RichtextFacet) + if err := t.DescriptionFacets[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.DescriptionFacets[i] pointer: %w", err) + } + } + + } + + } + } + // t.AcceptsInteractions (bool) (bool) + case "acceptsInteractions": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + + var val bool + switch extra { + case 20: + val = false + case 21: + val = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + t.AcceptsInteractions = &val + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *GraphListblock) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.listblock"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.graph.listblock")); err != nil { + return err + } + + // t.Subject (string) (string) + if len("subject") > 1000000 { + return xerrors.Errorf("Value in field \"subject\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + return err + } + if _, err := cw.WriteString(string("subject")); err != nil { + return err + } + + if len(t.Subject) > 1000000 { + return xerrors.Errorf("Value in field t.Subject was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Subject))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Subject)); err != nil { + return err + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + return nil +} + +func (t *GraphListblock) UnmarshalCBOR(r io.Reader) (err error) { + *t = GraphListblock{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("GraphListblock: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 9) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Subject (string) (string) + case "subject": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Subject = string(sval) + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *EmbedDefs_AspectRatio) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Width (int64) (int64) + if len("width") > 1000000 { + return xerrors.Errorf("Value in field \"width\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("width"))); err != nil { + return err + } + if _, err := cw.WriteString(string("width")); err != nil { + return err + } + + if t.Width >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Width)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Width-1)); err != nil { + return err + } + } + + // t.Height (int64) (int64) + if len("height") > 1000000 { + return xerrors.Errorf("Value in field \"height\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("height"))); err != nil { + return err + } + if _, err := cw.WriteString(string("height")); err != nil { + return err + } + + if t.Height >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Height)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Height-1)); err != nil { + return err + } + } + + return nil +} + +func (t *EmbedDefs_AspectRatio) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedDefs_AspectRatio{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("EmbedDefs_AspectRatio: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 6) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Width (int64) (int64) + case "width": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Width = int64(extraI) + } + // t.Height (int64) (int64) + case "height": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Height = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedThreadgate) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 5 + + if t.Allow == nil { + fieldCount-- + } + + if t.HiddenReplies == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.Post (string) (string) + if len("post") > 1000000 { + return xerrors.Errorf("Value in field \"post\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("post"))); err != nil { + return err + } + if _, err := cw.WriteString(string("post")); err != nil { + return err + } + + if len(t.Post) > 1000000 { + return xerrors.Errorf("Value in field t.Post was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Post))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Post)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.threadgate"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.feed.threadgate")); err != nil { + return err + } + + // t.Allow ([]*bsky.FeedThreadgate_Allow_Elem) (slice) + if t.Allow != nil { + + if len("allow") > 1000000 { + return xerrors.Errorf("Value in field \"allow\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("allow"))); err != nil { + return err + } + if _, err := cw.WriteString(string("allow")); err != nil { + return err + } + + if len(t.Allow) > 8192 { + return xerrors.Errorf("Slice value in field t.Allow was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Allow))); err != nil { + return err + } + for _, v := range t.Allow { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + + // t.HiddenReplies ([]string) (slice) + if t.HiddenReplies != nil { + + if len("hiddenReplies") > 1000000 { + return xerrors.Errorf("Value in field \"hiddenReplies\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("hiddenReplies"))); err != nil { + return err + } + if _, err := cw.WriteString(string("hiddenReplies")); err != nil { + return err + } + + if len(t.HiddenReplies) > 8192 { + return xerrors.Errorf("Slice value in field t.HiddenReplies was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.HiddenReplies))); err != nil { + return err + } + for _, v := range t.HiddenReplies { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } + return nil +} + +func (t *FeedThreadgate) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedThreadgate{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedThreadgate: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 13) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Post (string) (string) + case "post": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Post = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Allow ([]*bsky.FeedThreadgate_Allow_Elem) (slice) + case "allow": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Allow: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Allow = make([]*FeedThreadgate_Allow_Elem, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Allow[i] = new(FeedThreadgate_Allow_Elem) + if err := t.Allow[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Allow[i] pointer: %w", err) + } + } + + } + + } + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + // t.HiddenReplies ([]string) (slice) + case "hiddenReplies": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.HiddenReplies: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.HiddenReplies = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.HiddenReplies[i] = string(sval) + } + + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedThreadgate_ListRule) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.List (string) (string) + if len("list") > 1000000 { + return xerrors.Errorf("Value in field \"list\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("list"))); err != nil { + return err + } + if _, err := cw.WriteString(string("list")); err != nil { + return err + } + + if len(t.List) > 1000000 { + return xerrors.Errorf("Value in field t.List was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.List))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.List)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.threadgate#listRule"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.feed.threadgate#listRule")); err != nil { + return err + } + return nil +} + +func (t *FeedThreadgate_ListRule) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedThreadgate_ListRule{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedThreadgate_ListRule: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.List (string) (string) + case "list": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.List = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedThreadgate_MentionRule) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.threadgate#mentionRule"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.feed.threadgate#mentionRule")); err != nil { + return err + } + return nil +} + +func (t *FeedThreadgate_MentionRule) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedThreadgate_MentionRule{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedThreadgate_MentionRule: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedThreadgate_FollowerRule) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.threadgate#followerRule"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.feed.threadgate#followerRule")); err != nil { + return err + } + return nil +} + +func (t *FeedThreadgate_FollowerRule) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedThreadgate_FollowerRule{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedThreadgate_FollowerRule: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedThreadgate_FollowingRule) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.threadgate#followingRule"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.feed.threadgate#followingRule")); err != nil { + return err + } + return nil +} + +func (t *FeedThreadgate_FollowingRule) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedThreadgate_FollowingRule{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedThreadgate_FollowingRule: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *GraphStarterpack_FeedItem) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Uri (string) (string) + if len("uri") > 1000000 { + return xerrors.Errorf("Value in field \"uri\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { + return err + } + if _, err := cw.WriteString(string("uri")); err != nil { + return err + } + + if len(t.Uri) > 1000000 { + return xerrors.Errorf("Value in field t.Uri was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Uri)); err != nil { + return err + } + return nil +} + +func (t *GraphStarterpack_FeedItem) UnmarshalCBOR(r io.Reader) (err error) { + *t = GraphStarterpack_FeedItem{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("GraphStarterpack_FeedItem: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 3) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Uri (string) (string) + case "uri": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Uri = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *GraphStarterpack) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 7 + + if t.Description == nil { + fieldCount-- + } + + if t.DescriptionFacets == nil { + fieldCount-- + } + + if t.Feeds == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.List (string) (string) + if len("list") > 1000000 { + return xerrors.Errorf("Value in field \"list\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("list"))); err != nil { + return err + } + if _, err := cw.WriteString(string("list")); err != nil { + return err + } + + if len(t.List) > 1000000 { + return xerrors.Errorf("Value in field t.List was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.List))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.List)); err != nil { + return err + } + + // t.Name (string) (string) + if len("name") > 1000000 { + return xerrors.Errorf("Value in field \"name\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := cw.WriteString(string("name")); err != nil { + return err + } + + if len(t.Name) > 1000000 { + return xerrors.Errorf("Value in field t.Name was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Name)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.starterpack"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.graph.starterpack")); err != nil { + return err + } + + // t.Feeds ([]*bsky.GraphStarterpack_FeedItem) (slice) + if t.Feeds != nil { + + if len("feeds") > 1000000 { + return xerrors.Errorf("Value in field \"feeds\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("feeds"))); err != nil { + return err + } + if _, err := cw.WriteString(string("feeds")); err != nil { + return err + } + + if len(t.Feeds) > 8192 { + return xerrors.Errorf("Slice value in field t.Feeds was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Feeds))); err != nil { + return err + } + for _, v := range t.Feeds { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + } + + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + + // t.Description (string) (string) + if t.Description != nil { + + if len("description") > 1000000 { + return xerrors.Errorf("Value in field \"description\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("description"))); err != nil { + return err + } + if _, err := cw.WriteString(string("description")); err != nil { + return err + } + + if t.Description == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Description) > 1000000 { + return xerrors.Errorf("Value in field t.Description was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Description))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Description)); err != nil { + return err + } + } + } + + // t.DescriptionFacets ([]*bsky.RichtextFacet) (slice) + if t.DescriptionFacets != nil { + + if len("descriptionFacets") > 1000000 { + return xerrors.Errorf("Value in field \"descriptionFacets\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("descriptionFacets"))); err != nil { + return err + } + if _, err := cw.WriteString(string("descriptionFacets")); err != nil { + return err + } + + if len(t.DescriptionFacets) > 8192 { + return xerrors.Errorf("Slice value in field t.DescriptionFacets was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.DescriptionFacets))); err != nil { + return err + } + for _, v := range t.DescriptionFacets { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + } + return nil +} + +func (t *GraphStarterpack) UnmarshalCBOR(r io.Reader) (err error) { + *t = GraphStarterpack{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("GraphStarterpack: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 17) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.List (string) (string) + case "list": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.List = string(sval) + } + // t.Name (string) (string) + case "name": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Name = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.Feeds ([]*bsky.GraphStarterpack_FeedItem) (slice) + case "feeds": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Feeds: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Feeds = make([]*GraphStarterpack_FeedItem, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Feeds[i] = new(GraphStarterpack_FeedItem) + if err := t.Feeds[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Feeds[i] pointer: %w", err) + } + } + + } + + } + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + // t.Description (string) (string) + case "description": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Description = (*string)(&sval) + } + } + // t.DescriptionFacets ([]*bsky.RichtextFacet) (slice) + case "descriptionFacets": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.DescriptionFacets: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.DescriptionFacets = make([]*RichtextFacet, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.DescriptionFacets[i] = new(RichtextFacet) + if err := t.DescriptionFacets[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.DescriptionFacets[i] pointer: %w", err) + } + } + + } + + } + } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *EmbedExternal_External) MarshalCBOR(w io.Writer) error { +func (t *LabelerService) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) + fieldCount := 7 - if _, err := cw.Write([]byte{165}); err != nil { - return err + if t.Labels == nil { + fieldCount-- } - // t.Uri (string) (string) - if len("uri") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"uri\" was too long") + if t.ReasonTypes == nil { + fieldCount-- } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("uri")); err != nil { - return err + if t.SubjectCollections == nil { + fieldCount-- } - if len(t.Uri) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Uri was too long") + if t.SubjectTypes == nil { + fieldCount-- } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Uri)); err != nil { + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } - // t.Thumb (util.Blob) (struct) - if len("thumb") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"thumb\" was too long") + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("thumb"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("thumb")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } - if err := t.Thumb.MarshalCBOR(cw); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.labeler.service"))); err != nil { return err } - - // t.Title (string) (string) - if len("title") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"title\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("title"))); err != nil { + if _, err := cw.WriteString(string("app.bsky.labeler.service")); err != nil { return err } - if _, err := io.WriteString(w, string("title")); err != nil { - return err + + // t.Labels (bsky.LabelerService_Labels) (struct) + if t.Labels != nil { + + if len("labels") > 1000000 { + return xerrors.Errorf("Value in field \"labels\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("labels"))); err != nil { + return err + } + if _, err := cw.WriteString(string("labels")); err != nil { + return err + } + + if err := t.Labels.MarshalCBOR(cw); err != nil { + return err + } } - if len(t.Title) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Title was too long") + // t.Policies (bsky.LabelerDefs_LabelerPolicies) (struct) + if len("policies") > 1000000 { + return xerrors.Errorf("Value in field \"policies\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Title))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("policies"))); err != nil { return err } - if _, err := io.WriteString(w, string(t.Title)); err != nil { + if _, err := cw.WriteString(string("policies")); err != nil { return err } - // t.Description (string) (string) - if len("description") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"description\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("description"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("description")); err != nil { + if err := t.Policies.MarshalCBOR(cw); err != nil { return err } - if len(t.Description) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Description was too long") + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Description))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { return err } - if _, err := io.WriteString(w, string(t.Description)); err != nil { + if _, err := cw.WriteString(string("createdAt")); err != nil { return err } - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { return err } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") + // t.ReasonTypes ([]*string) (slice) + if t.ReasonTypes != nil { + + if len("reasonTypes") > 1000000 { + return xerrors.Errorf("Value in field \"reasonTypes\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("reasonTypes"))); err != nil { + return err + } + if _, err := cw.WriteString(string("reasonTypes")); err != nil { + return err + } + + if len(t.ReasonTypes) > 8192 { + return xerrors.Errorf("Slice value in field t.ReasonTypes was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.ReasonTypes))); err != nil { + return err + } + for _, v := range t.ReasonTypes { + if v == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*v))); err != nil { + return err + } + if _, err := cw.WriteString(string(*v)); err != nil { + return err + } + } + + } } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { - return err + // t.SubjectTypes ([]*string) (slice) + if t.SubjectTypes != nil { + + if len("subjectTypes") > 1000000 { + return xerrors.Errorf("Value in field \"subjectTypes\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subjectTypes"))); err != nil { + return err + } + if _, err := cw.WriteString(string("subjectTypes")); err != nil { + return err + } + + if len(t.SubjectTypes) > 8192 { + return xerrors.Errorf("Slice value in field t.SubjectTypes was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.SubjectTypes))); err != nil { + return err + } + for _, v := range t.SubjectTypes { + if v == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*v))); err != nil { + return err + } + if _, err := cw.WriteString(string(*v)); err != nil { + return err + } + } + + } } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { - return err + + // t.SubjectCollections ([]string) (slice) + if t.SubjectCollections != nil { + + if len("subjectCollections") > 1000000 { + return xerrors.Errorf("Value in field \"subjectCollections\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subjectCollections"))); err != nil { + return err + } + if _, err := cw.WriteString(string("subjectCollections")); err != nil { + return err + } + + if len(t.SubjectCollections) > 8192 { + return xerrors.Errorf("Slice value in field t.SubjectCollections was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.SubjectCollections))); err != nil { + return err + } + for _, v := range t.SubjectCollections { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } } return nil } -func (t *EmbedExternal_External) UnmarshalCBOR(r io.Reader) (err error) { - *t = EmbedExternal_External{} +func (t *LabelerService) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelerService{} cr := cbg.NewCborReader(r) @@ -2000,175 +7408,329 @@ func (t *EmbedExternal_External) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("EmbedExternal_External: map struct too large (%d)", extra) + return fmt.Errorf("LabelerService: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 18) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.Uri (string) (string) - case "uri": + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - t.Uri = string(sval) + t.LexiconTypeID = string(sval) + } + // t.Labels (bsky.LabelerService_Labels) (struct) + case "labels": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Labels = new(LabelerService_Labels) + if err := t.Labels.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Labels pointer: %w", err) + } + } + + } + // t.Policies (bsky.LabelerDefs_LabelerPolicies) (struct) + case "policies": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Policies = new(LabelerDefs_LabelerPolicies) + if err := t.Policies.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Policies pointer: %w", err) + } + } + + } + // t.CreatedAt (string) (string) + case "createdAt": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + // t.ReasonTypes ([]*string) (slice) + case "reasonTypes": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.ReasonTypes: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.ReasonTypes = make([]*string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.ReasonTypes[i] = (*string)(&sval) + } + } + + } + } + // t.SubjectTypes ([]*string) (slice) + case "subjectTypes": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.SubjectTypes: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") } - // t.Thumb (util.Blob) (struct) - case "thumb": - { + if extra > 0 { + t.SubjectTypes = make([]*string, extra) + } - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Thumb = new(util.Blob) - if err := t.Thumb.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Thumb pointer: %w", err) + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.SubjectTypes[i] = (*string)(&sval) + } } - } + } } - // t.Title (string) (string) - case "title": + // t.SubjectCollections ([]string) (slice) + case "subjectCollections": - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } - t.Title = string(sval) + if extra > 8192 { + return fmt.Errorf("t.SubjectCollections: array too large (%d)", extra) } - // t.Description (string) (string) - case "description": - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } - t.Description = string(sval) + if extra > 0 { + t.SubjectCollections = make([]string, extra) } - // t.LexiconTypeID (string) (string) - case "LexiconTypeID": - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.SubjectCollections[i] = string(sval) + } - t.LexiconTypeID = string(sval) + } } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *EmbedImages_Image) MarshalCBOR(w io.Writer) error { +func (t *LabelerDefs_LabelerPolicies) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) + fieldCount := 2 - if _, err := cw.Write([]byte{163}); err != nil { - return err - } - - // t.Alt (string) (string) - if len("alt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"alt\" was too long") + if t.LabelValueDefinitions == nil { + fieldCount-- } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("alt"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("alt")); err != nil { + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } - if len(t.Alt) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Alt was too long") + // t.LabelValues ([]*string) (slice) + if len("labelValues") > 1000000 { + return xerrors.Errorf("Value in field \"labelValues\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Alt))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("labelValues"))); err != nil { return err } - if _, err := io.WriteString(w, string(t.Alt)); err != nil { + if _, err := cw.WriteString(string("labelValues")); err != nil { return err } - // t.Image (util.Blob) (struct) - if len("image") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"image\" was too long") + if len(t.LabelValues) > 8192 { + return xerrors.Errorf("Slice value in field t.LabelValues was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("image"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("image")); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.LabelValues))); err != nil { return err } + for _, v := range t.LabelValues { + if v == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } - if err := t.Image.MarshalCBOR(cw); err != nil { - return err - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*v))); err != nil { + return err + } + if _, err := cw.WriteString(string(*v)); err != nil { + return err + } + } - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { - return err - } + // t.LabelValueDefinitions ([]*atproto.LabelDefs_LabelValueDefinition) (slice) + if t.LabelValueDefinitions != nil { - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") - } + if len("labelValueDefinitions") > 1000000 { + return xerrors.Errorf("Value in field \"labelValueDefinitions\" was too long") + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { - return err + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("labelValueDefinitions"))); err != nil { + return err + } + if _, err := cw.WriteString(string("labelValueDefinitions")); err != nil { + return err + } + + if len(t.LabelValueDefinitions) > 8192 { + return xerrors.Errorf("Slice value in field t.LabelValueDefinitions was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.LabelValueDefinitions))); err != nil { + return err + } + for _, v := range t.LabelValueDefinitions { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } } return nil } -func (t *EmbedImages_Image) UnmarshalCBOR(r io.Reader) (err error) { - *t = EmbedImages_Image{} +func (t *LabelerDefs_LabelerPolicies) UnmarshalCBOR(r io.Reader) (err error) { + *t = LabelerDefs_LabelerPolicies{} cr := cbg.NewCborReader(r) @@ -2187,149 +7749,281 @@ func (t *EmbedImages_Image) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("EmbedImages_Image: map struct too large (%d)", extra) + return fmt.Errorf("LabelerDefs_LabelerPolicies: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 21) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.Alt (string) (string) - case "alt": + switch string(nameBuf[:nameLen]) { + // t.LabelValues ([]*string) (slice) + case "labelValues": - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } - t.Alt = string(sval) + if extra > 8192 { + return fmt.Errorf("t.LabelValues: array too large (%d)", extra) } - // t.Image (util.Blob) (struct) - case "image": - { + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Image = new(util.Blob) - if err := t.Image.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Image pointer: %w", err) + if extra > 0 { + t.LabelValues = make([]*string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LabelValues[i] = (*string)(&sval) + } } + } + } + // t.LabelValueDefinitions ([]*atproto.LabelDefs_LabelValueDefinition) (slice) + case "labelValueDefinitions": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.LabelValueDefinitions: array too large (%d)", extra) } - // t.LexiconTypeID (string) (string) - case "LexiconTypeID": - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } - t.LexiconTypeID = string(sval) + if extra > 0 { + t.LabelValueDefinitions = make([]*atproto.LabelDefs_LabelValueDefinition, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.LabelValueDefinitions[i] = new(atproto.LabelDefs_LabelValueDefinition) + if err := t.LabelValueDefinitions[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.LabelValueDefinitions[i] pointer: %w", err) + } + } + + } + + } } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *GraphFollow) MarshalCBOR(w io.Writer) error { +func (t *EmbedVideo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) + fieldCount := 5 - if _, err := cw.Write([]byte{163}); err != nil { + if t.Alt == nil { + fieldCount-- + } + + if t.AspectRatio == nil { + fieldCount-- + } + + if t.Captions == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } + // t.Alt (string) (string) + if t.Alt != nil { + + if len("alt") > 1000000 { + return xerrors.Errorf("Value in field \"alt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("alt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("alt")); err != nil { + return err + } + + if t.Alt == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Alt) > 1000000 { + return xerrors.Errorf("Value in field t.Alt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Alt))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Alt)); err != nil { + return err + } + } + } + // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("$type")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.follow"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.embed.video"))); err != nil { return err } - if _, err := io.WriteString(w, string("app.bsky.graph.follow")); err != nil { + if _, err := cw.WriteString(string("app.bsky.embed.video")); err != nil { return err } - // t.Subject (schemagen.ActorRef) (struct) - if len("subject") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"subject\" was too long") + // t.Video (util.LexBlob) (struct) + if len("video") > 1000000 { + return xerrors.Errorf("Value in field \"video\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("video"))); err != nil { return err } - if _, err := io.WriteString(w, string("subject")); err != nil { + if _, err := cw.WriteString(string("video")); err != nil { return err } - if err := t.Subject.MarshalCBOR(cw); err != nil { + if err := t.Video.MarshalCBOR(cw); err != nil { return err } - // t.CreatedAt (string) (string) - if len("createdAt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"createdAt\" was too long") - } + // t.Captions ([]*bsky.EmbedVideo_Caption) (slice) + if t.Captions != nil { - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("createdAt")); err != nil { - return err - } + if len("captions") > 1000000 { + return xerrors.Errorf("Value in field \"captions\" was too long") + } - if len(t.CreatedAt) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.CreatedAt was too long") - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("captions"))); err != nil { + return err + } + if _, err := cw.WriteString(string("captions")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { - return err + if len(t.Captions) > 8192 { + return xerrors.Errorf("Slice value in field t.Captions was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Captions))); err != nil { + return err + } + for _, v := range t.Captions { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } } - if _, err := io.WriteString(w, string(t.CreatedAt)); err != nil { - return err + + // t.AspectRatio (bsky.EmbedDefs_AspectRatio) (struct) + if t.AspectRatio != nil { + + if len("aspectRatio") > 1000000 { + return xerrors.Errorf("Value in field \"aspectRatio\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("aspectRatio"))); err != nil { + return err + } + if _, err := cw.WriteString(string("aspectRatio")); err != nil { + return err + } + + if err := t.AspectRatio.MarshalCBOR(cw); err != nil { + return err + } } return nil } -func (t *GraphFollow) UnmarshalCBOR(r io.Reader) (err error) { - *t = GraphFollow{} +func (t *EmbedVideo) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedVideo{} cr := cbg.NewCborReader(r) @@ -2348,37 +8042,61 @@ func (t *GraphFollow) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("GraphFollow: map struct too large (%d)", extra) + return fmt.Errorf("EmbedVideo: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.LexiconTypeID (string) (string) + switch string(nameBuf[:nameLen]) { + // t.Alt (string) (string) + case "alt": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Alt = (*string)(&sval) + } + } + // t.LexiconTypeID (string) (string) case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.LexiconTypeID = string(sval) } - // t.Subject (schemagen.ActorRef) (struct) - case "subject": + // t.Video (util.LexBlob) (struct) + case "video": { @@ -2390,34 +8108,94 @@ func (t *GraphFollow) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Subject = new(ActorRef) - if err := t.Subject.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Subject pointer: %w", err) + t.Video = new(util.LexBlob) + if err := t.Video.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Video pointer: %w", err) } } } - // t.CreatedAt (string) (string) - case "createdAt": + // t.Captions ([]*bsky.EmbedVideo_Caption) (slice) + case "captions": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Captions: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Captions = make([]*EmbedVideo_Caption, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Captions[i] = new(EmbedVideo_Caption) + if err := t.Captions[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Captions[i] pointer: %w", err) + } + } + + } + + } + } + // t.AspectRatio (bsky.EmbedDefs_AspectRatio) (struct) + case "aspectRatio": { - sval, err := cbg.ReadString(cr) + + b, err := cr.ReadByte() if err != nil { return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.AspectRatio = new(EmbedDefs_AspectRatio) + if err := t.AspectRatio.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.AspectRatio pointer: %w", err) + } + } - t.CreatedAt = string(sval) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *ActorRef) MarshalCBOR(w io.Writer) error { +func (t *EmbedVideo_Caption) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -2425,83 +8203,53 @@ func (t *ActorRef) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{163}); err != nil { - return err - } - - // t.Did (string) (string) - if len("did") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"did\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("did")); err != nil { - return err - } - - if len(t.Did) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Did was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Did))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Did)); err != nil { + if _, err := cw.Write([]byte{162}); err != nil { return err } - // t.LexiconTypeID (string) (string) - if len("LexiconTypeID") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") + // t.File (util.LexBlob) (struct) + if len("file") > 1000000 { + return xerrors.Errorf("Value in field \"file\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("file"))); err != nil { return err } - if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { + if _, err := cw.WriteString(string("file")); err != nil { return err } - if len(t.LexiconTypeID) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.LexiconTypeID was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { + if err := t.File.MarshalCBOR(cw); err != nil { return err } - // t.DeclarationCid (string) (string) - if len("declarationCid") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"declarationCid\" was too long") + // t.Lang (string) (string) + if len("lang") > 1000000 { + return xerrors.Errorf("Value in field \"lang\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("declarationCid"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("lang"))); err != nil { return err } - if _, err := io.WriteString(w, string("declarationCid")); err != nil { + if _, err := cw.WriteString(string("lang")); err != nil { return err } - if len(t.DeclarationCid) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.DeclarationCid was too long") + if len(t.Lang) > 1000000 { + return xerrors.Errorf("Value in field t.Lang was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.DeclarationCid))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Lang))); err != nil { return err } - if _, err := io.WriteString(w, string(t.DeclarationCid)); err != nil { + if _, err := cw.WriteString(string(t.Lang)); err != nil { return err } return nil } -func (t *ActorRef) UnmarshalCBOR(r io.Reader) (err error) { - *t = ActorRef{} +func (t *EmbedVideo_Caption) UnmarshalCBOR(r io.Reader) (err error) { + *t = EmbedVideo_Caption{} cr := cbg.NewCborReader(r) @@ -2520,185 +8268,224 @@ func (t *ActorRef) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("ActorRef: map struct too large (%d)", extra) + return fmt.Errorf("EmbedVideo_Caption: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 4) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.Did (string) (string) - case "did": + switch string(nameBuf[:nameLen]) { + // t.File (util.LexBlob) (struct) + case "file": { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - t.Did = string(sval) - } - // t.LexiconTypeID (string) (string) - case "LexiconTypeID": - - { - sval, err := cbg.ReadString(cr) + b, err := cr.ReadByte() if err != nil { return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.File = new(util.LexBlob) + if err := t.File.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.File pointer: %w", err) + } + } - t.LexiconTypeID = string(sval) } - // t.DeclarationCid (string) (string) - case "declarationCid": + // t.Lang (string) (string) + case "lang": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - t.DeclarationCid = string(sval) + t.Lang = string(sval) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *ActorProfile) MarshalCBOR(w io.Writer) error { +func (t *FeedPostgate) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) + fieldCount := 5 - if _, err := cw.Write([]byte{165}); err != nil { - return err + if t.DetachedEmbeddingUris == nil { + fieldCount-- } - // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"$type\" was too long") + if t.EmbeddingRules == nil { + fieldCount-- } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } - if _, err := io.WriteString(w, string("$type")); err != nil { - return err + + // t.Post (string) (string) + if len("post") > 1000000 { + return xerrors.Errorf("Value in field \"post\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.actor.profile"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("post"))); err != nil { return err } - if _, err := io.WriteString(w, string("app.bsky.actor.profile")); err != nil { + if _, err := cw.WriteString(string("post")); err != nil { return err } - // t.Avatar (util.Blob) (struct) - if len("avatar") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"avatar\" was too long") + if len(t.Post) > 1000000 { + return xerrors.Errorf("Value in field t.Post was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("avatar"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Post))); err != nil { return err } - if _, err := io.WriteString(w, string("avatar")); err != nil { + if _, err := cw.WriteString(string(t.Post)); err != nil { return err } - if err := t.Avatar.MarshalCBOR(cw); err != nil { - return err + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") } - // t.Banner (util.Blob) (struct) - if len("banner") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"banner\" was too long") + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("banner"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.postgate"))); err != nil { return err } - if _, err := io.WriteString(w, string("banner")); err != nil { + if _, err := cw.WriteString(string("app.bsky.feed.postgate")); err != nil { return err } - if err := t.Banner.MarshalCBOR(cw); err != nil { + // t.CreatedAt (string) (string) + if len("createdAt") > 1000000 { + return xerrors.Errorf("Value in field \"createdAt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { + return err + } + if _, err := cw.WriteString(string("createdAt")); err != nil { return err } - // t.Description (string) (string) - if len("description") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"description\" was too long") + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("description"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { return err } - if _, err := io.WriteString(w, string("description")); err != nil { + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } - if t.Description == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { + // t.EmbeddingRules ([]*bsky.FeedPostgate_EmbeddingRules_Elem) (slice) + if t.EmbeddingRules != nil { + + if len("embeddingRules") > 1000000 { + return xerrors.Errorf("Value in field \"embeddingRules\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("embeddingRules"))); err != nil { return err } - } else { - if len(*t.Description) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Description was too long") + if _, err := cw.WriteString(string("embeddingRules")); err != nil { + return err } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Description))); err != nil { - return err + if len(t.EmbeddingRules) > 8192 { + return xerrors.Errorf("Slice value in field t.EmbeddingRules was too long") } - if _, err := io.WriteString(w, string(*t.Description)); err != nil { + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.EmbeddingRules))); err != nil { return err } - } + for _, v := range t.EmbeddingRules { + if err := v.MarshalCBOR(cw); err != nil { + return err + } - // t.DisplayName (string) (string) - if len("displayName") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"displayName\" was too long") + } } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("displayName"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("displayName")); err != nil { - return err - } + // t.DetachedEmbeddingUris ([]string) (slice) + if t.DetachedEmbeddingUris != nil { - if len(t.DisplayName) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.DisplayName was too long") - } + if len("detachedEmbeddingUris") > 1000000 { + return xerrors.Errorf("Value in field \"detachedEmbeddingUris\" was too long") + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.DisplayName))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.DisplayName)); err != nil { - return err + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("detachedEmbeddingUris"))); err != nil { + return err + } + if _, err := cw.WriteString(string("detachedEmbeddingUris")); err != nil { + return err + } + + if len(t.DetachedEmbeddingUris) > 8192 { + return xerrors.Errorf("Slice value in field t.DetachedEmbeddingUris was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.DetachedEmbeddingUris))); err != nil { + return err + } + for _, v := range t.DetachedEmbeddingUris { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } } return nil } -func (t *ActorProfile) UnmarshalCBOR(r io.Reader) (err error) { - *t = ActorProfile{} +func (t *FeedPostgate) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedPostgate{} cr := cbg.NewCborReader(r) @@ -2717,117 +8504,161 @@ func (t *ActorProfile) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("ActorProfile: map struct too large (%d)", extra) + return fmt.Errorf("FeedPostgate: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 21) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { - // t.LexiconTypeID (string) (string) - case "$type": + switch string(nameBuf[:nameLen]) { + // t.Post (string) (string) + case "post": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - t.LexiconTypeID = string(sval) + t.Post = string(sval) } - // t.Avatar (util.Blob) (struct) - case "avatar": + // t.LexiconTypeID (string) (string) + case "$type": { - - b, err := cr.ReadByte() + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Avatar = new(util.Blob) - if err := t.Avatar.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Avatar pointer: %w", err) - } - } + t.LexiconTypeID = string(sval) } - // t.Banner (util.Blob) (struct) - case "banner": + // t.CreatedAt (string) (string) + case "createdAt": { - - b, err := cr.ReadByte() + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Banner = new(util.Blob) - if err := t.Banner.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Banner pointer: %w", err) - } - } + t.CreatedAt = string(sval) } - // t.Description (string) (string) - case "description": + // t.EmbeddingRules ([]*bsky.FeedPostgate_EmbeddingRules_Elem) (slice) + case "embeddingRules": - { - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.EmbeddingRules: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.EmbeddingRules = make([]*FeedPostgate_EmbeddingRules_Elem, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.EmbeddingRules[i] = new(FeedPostgate_EmbeddingRules_Elem) + if err := t.EmbeddingRules[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.EmbeddingRules[i] pointer: %w", err) + } + } - sval, err := cbg.ReadString(cr) - if err != nil { - return err } - t.Description = (*string)(&sval) } } - // t.DisplayName (string) (string) - case "displayName": + // t.DetachedEmbeddingUris ([]string) (slice) + case "detachedEmbeddingUris": - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.DetachedEmbeddingUris: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.DetachedEmbeddingUris = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.DetachedEmbeddingUris[i] = string(sval) + } - t.DisplayName = string(sval) + } } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *SystemDeclaration) MarshalCBOR(w io.Writer) error { +func (t *FeedPostgate_DisableRule) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -2835,56 +8666,33 @@ func (t *SystemDeclaration) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{162}); err != nil { + if _, err := cw.Write([]byte{161}); err != nil { return err } // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("$type")); err != nil { - return err - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.system.declaration"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("app.bsky.system.declaration")); err != nil { - return err - } - - // t.ActorType (string) (string) - if len("actorType") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"actorType\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actorType"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("actorType")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } - if len(t.ActorType) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.ActorType was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.ActorType))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.feed.postgate#disableRule"))); err != nil { return err } - if _, err := io.WriteString(w, string(t.ActorType)); err != nil { + if _, err := cw.WriteString(string("app.bsky.feed.postgate#disableRule")); err != nil { return err } return nil } -func (t *SystemDeclaration) UnmarshalCBOR(r io.Reader) (err error) { - *t = SystemDeclaration{} +func (t *FeedPostgate_DisableRule) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedPostgate_DisableRule{} cr := cbg.NewCborReader(r) @@ -2903,56 +8711,50 @@ func (t *SystemDeclaration) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("SystemDeclaration: map struct too large (%d)", extra) + return fmt.Errorf("FeedPostgate_DisableRule: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 5) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.LexiconTypeID = string(sval) } - // t.ActorType (string) (string) - case "actorType": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.ActorType = string(sval) - } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *GraphAssertion) MarshalCBOR(w io.Writer) error { +func (t *GraphVerification) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -2960,95 +8762,125 @@ func (t *GraphAssertion) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{164}); err != nil { + if _, err := cw.Write([]byte{165}); err != nil { return err } // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("$type")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.assertion"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.verification"))); err != nil { return err } - if _, err := io.WriteString(w, string("app.bsky.graph.assertion")); err != nil { + if _, err := cw.WriteString(string("app.bsky.graph.verification")); err != nil { return err } - // t.Subject (schemagen.ActorRef) (struct) - if len("subject") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"subject\" was too long") + // t.Handle (string) (string) + if len("handle") > 1000000 { + return xerrors.Errorf("Value in field \"handle\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("handle"))); err != nil { return err } - if _, err := io.WriteString(w, string("subject")); err != nil { + if _, err := cw.WriteString(string("handle")); err != nil { return err } - if err := t.Subject.MarshalCBOR(cw); err != nil { + if len(t.Handle) > 1000000 { + return xerrors.Errorf("Value in field t.Handle was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Handle))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Handle)); err != nil { return err } - // t.Assertion (string) (string) - if len("assertion") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"assertion\" was too long") + // t.Subject (string) (string) + if len("subject") > 1000000 { + return xerrors.Errorf("Value in field \"subject\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("assertion"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { return err } - if _, err := io.WriteString(w, string("assertion")); err != nil { + if _, err := cw.WriteString(string("subject")); err != nil { return err } - if len(t.Assertion) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Assertion was too long") + if len(t.Subject) > 1000000 { + return xerrors.Errorf("Value in field t.Subject was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Assertion))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Subject))); err != nil { return err } - if _, err := io.WriteString(w, string(t.Assertion)); err != nil { + if _, err := cw.WriteString(string(t.Subject)); err != nil { return err } // t.CreatedAt (string) (string) - if len("createdAt") > cbg.MaxLength { + if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { return err } - if _, err := io.WriteString(w, string("createdAt")); err != nil { + if _, err := cw.WriteString(string("createdAt")); err != nil { return err } - if len(t.CreatedAt) > cbg.MaxLength { + if len(t.CreatedAt) > 1000000 { return xerrors.Errorf("Value in field t.CreatedAt was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { return err } - if _, err := io.WriteString(w, string(t.CreatedAt)); err != nil { + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + + // t.DisplayName (string) (string) + if len("displayName") > 1000000 { + return xerrors.Errorf("Value in field \"displayName\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("displayName"))); err != nil { + return err + } + if _, err := cw.WriteString(string("displayName")); err != nil { + return err + } + + if len(t.DisplayName) > 1000000 { + return xerrors.Errorf("Value in field t.DisplayName was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.DisplayName))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.DisplayName)); err != nil { return err } return nil } -func (t *GraphAssertion) UnmarshalCBOR(r io.Reader) (err error) { - *t = GraphAssertion{} +func (t *GraphVerification) UnmarshalCBOR(r io.Reader) (err error) { + *t = GraphVerification{} cr := cbg.NewCborReader(r) @@ -3067,176 +8899,234 @@ func (t *GraphAssertion) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("GraphAssertion: map struct too large (%d)", extra) + return fmt.Errorf("GraphVerification: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 11) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.LexiconTypeID = string(sval) } - // t.Subject (schemagen.ActorRef) (struct) - case "subject": + // t.Handle (string) (string) + case "handle": { - - b, err := cr.ReadByte() + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Subject = new(ActorRef) - if err := t.Subject.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Subject pointer: %w", err) - } - } + t.Handle = string(sval) } - // t.Assertion (string) (string) - case "assertion": + // t.Subject (string) (string) + case "subject": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - t.Assertion = string(sval) + t.Subject = string(sval) } // t.CreatedAt (string) (string) case "createdAt": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.CreatedAt = string(sval) } + // t.DisplayName (string) (string) + case "displayName": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.DisplayName = string(sval) + } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } return nil } -func (t *GraphConfirmation) MarshalCBOR(w io.Writer) error { +func (t *ActorStatus) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) + fieldCount := 5 - if _, err := cw.Write([]byte{164}); err != nil { + if t.DurationMinutes == nil { + fieldCount-- + } + + if t.Embed == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } // t.LexiconTypeID (string) (string) - if len("$type") > cbg.MaxLength { + if len("$type") > 1000000 { return xerrors.Errorf("Value in field \"$type\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { return err } - if _, err := io.WriteString(w, string("$type")); err != nil { + if _, err := cw.WriteString(string("$type")); err != nil { return err } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.graph.confirmation"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.actor.status"))); err != nil { return err } - if _, err := io.WriteString(w, string("app.bsky.graph.confirmation")); err != nil { + if _, err := cw.WriteString(string("app.bsky.actor.status")); err != nil { return err } - // t.Assertion (schemagen.RepoStrongRef) (struct) - if len("assertion") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"assertion\" was too long") + // t.Embed (bsky.ActorStatus_Embed) (struct) + if t.Embed != nil { + + if len("embed") > 1000000 { + return xerrors.Errorf("Value in field \"embed\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("embed"))); err != nil { + return err + } + if _, err := cw.WriteString(string("embed")); err != nil { + return err + } + + if err := t.Embed.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.Status (string) (string) + if len("status") > 1000000 { + return xerrors.Errorf("Value in field \"status\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("assertion"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("status"))); err != nil { return err } - if _, err := io.WriteString(w, string("assertion")); err != nil { + if _, err := cw.WriteString(string("status")); err != nil { return err } - if err := t.Assertion.MarshalCBOR(cw); err != nil { + if len(t.Status) > 1000000 { + return xerrors.Errorf("Value in field t.Status was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Status))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Status)); err != nil { return err } // t.CreatedAt (string) (string) - if len("createdAt") > cbg.MaxLength { + if len("createdAt") > 1000000 { return xerrors.Errorf("Value in field \"createdAt\" was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { return err } - if _, err := io.WriteString(w, string("createdAt")); err != nil { + if _, err := cw.WriteString(string("createdAt")); err != nil { return err } - if len(t.CreatedAt) > cbg.MaxLength { + if len(t.CreatedAt) > 1000000 { return xerrors.Errorf("Value in field t.CreatedAt was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { return err } - if _, err := io.WriteString(w, string(t.CreatedAt)); err != nil { + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } - // t.Originator (schemagen.ActorRef) (struct) - if len("originator") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"originator\" was too long") - } + // t.DurationMinutes (int64) (int64) + if t.DurationMinutes != nil { - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("originator"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("originator")); err != nil { - return err - } + if len("durationMinutes") > 1000000 { + return xerrors.Errorf("Value in field \"durationMinutes\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("durationMinutes"))); err != nil { + return err + } + if _, err := cw.WriteString(string("durationMinutes")); err != nil { + return err + } + + if t.DurationMinutes == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if *t.DurationMinutes >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(*t.DurationMinutes)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-*t.DurationMinutes-1)); err != nil { + return err + } + } + } - if err := t.Originator.MarshalCBOR(cw); err != nil { - return err } return nil } -func (t *GraphConfirmation) UnmarshalCBOR(r io.Reader) (err error) { - *t = GraphConfirmation{} +func (t *ActorStatus) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorStatus{} cr := cbg.NewCborReader(r) @@ -3255,37 +9145,40 @@ func (t *GraphConfirmation) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("GraphConfirmation: map struct too large (%d)", extra) + return fmt.Errorf("ActorStatus: map struct too large (%d)", extra) } - var name string n := extra + nameBuf := make([]byte, 15) for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(cr) - if err != nil { + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { return err } - - name = string(sval) + continue } - switch name { + switch string(nameBuf[:nameLen]) { // t.LexiconTypeID (string) (string) case "$type": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.LexiconTypeID = string(sval) } - // t.Assertion (schemagen.RepoStrongRef) (struct) - case "assertion": + // t.Embed (bsky.ActorStatus_Embed) (struct) + case "embed": { @@ -3297,27 +9190,37 @@ func (t *GraphConfirmation) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Assertion = new(schemagen.RepoStrongRef) - if err := t.Assertion.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Assertion pointer: %w", err) + t.Embed = new(ActorStatus_Embed) + if err := t.Embed.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Embed pointer: %w", err) } } } + // t.Status (string) (string) + case "status": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Status = string(sval) + } // t.CreatedAt (string) (string) case "createdAt": { - sval, err := cbg.ReadString(cr) + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } t.CreatedAt = string(sval) } - // t.Originator (schemagen.ActorRef) (struct) - case "originator": - + // t.DurationMinutes (int64) (int64) + case "durationMinutes": { b, err := cr.ReadByte() @@ -3328,17 +9231,166 @@ func (t *GraphConfirmation) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Originator = new(ActorRef) - if err := t.Originator.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Originator pointer: %w", err) + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) } + + t.DurationMinutes = (*int64)(&extraI) + } + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *NotificationDeclaration) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("app.bsky.notification.declaration"))); err != nil { + return err + } + if _, err := cw.WriteString(string("app.bsky.notification.declaration")); err != nil { + return err + } + + // t.AllowSubscriptions (string) (string) + if len("allowSubscriptions") > 1000000 { + return xerrors.Errorf("Value in field \"allowSubscriptions\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("allowSubscriptions"))); err != nil { + return err + } + if _, err := cw.WriteString(string("allowSubscriptions")); err != nil { + return err + } + + if len(t.AllowSubscriptions) > 1000000 { + return xerrors.Errorf("Value in field t.AllowSubscriptions was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.AllowSubscriptions))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.AllowSubscriptions)); err != nil { + return err + } + return nil +} + +func (t *NotificationDeclaration) UnmarshalCBOR(r io.Reader) (err error) { + *t = NotificationDeclaration{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("NotificationDeclaration: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 18) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.AllowSubscriptions (string) (string) + case "allowSubscriptions": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err } + t.AllowSubscriptions = string(sval) } default: // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } } } diff --git a/api/bsky/contactdefs.go b/api/bsky/contactdefs.go new file mode 100644 index 000000000..cd260d8eb --- /dev/null +++ b/api/bsky/contactdefs.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.defs + +package bsky + +// ContactDefs_MatchAndContactIndex is a "matchAndContactIndex" in the app.bsky.contact.defs schema. +// +// Associates a profile with the positional index of the contact import input in the call to `app.bsky.contact.importContacts`, so clients can know which phone caused a particular match. +type ContactDefs_MatchAndContactIndex struct { + // contactIndex: The index of this match in the import contact input. + ContactIndex int64 `json:"contactIndex" cborgen:"contactIndex"` + // match: Profile of the matched user. + Match *ActorDefs_ProfileView `json:"match" cborgen:"match"` +} + +// ContactDefs_Notification is a "notification" in the app.bsky.contact.defs schema. +// +// A stash object to be sent via bsync representing a notification to be created. +type ContactDefs_Notification struct { + // from: The DID of who this notification comes from. + From string `json:"from" cborgen:"from"` + // to: The DID of who this notification should go to. + To string `json:"to" cborgen:"to"` +} + +// ContactDefs_SyncStatus is a "syncStatus" in the app.bsky.contact.defs schema. +type ContactDefs_SyncStatus struct { + // matchesCount: Number of existing contact matches resulting of the user imports and of their imported contacts having imported the user. Matches stop being counted when the user either follows the matched contact or dismisses the match. + MatchesCount int64 `json:"matchesCount" cborgen:"matchesCount"` + // syncedAt: Last date when contacts where imported. + SyncedAt string `json:"syncedAt" cborgen:"syncedAt"` +} diff --git a/api/bsky/contactdismissMatch.go b/api/bsky/contactdismissMatch.go new file mode 100644 index 000000000..136c9a8bd --- /dev/null +++ b/api/bsky/contactdismissMatch.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.dismissMatch + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ContactDismissMatch_Input is the input argument to a app.bsky.contact.dismissMatch call. +type ContactDismissMatch_Input struct { + // subject: The subject's DID to dismiss the match with. + Subject string `json:"subject" cborgen:"subject"` +} + +// ContactDismissMatch_Output is the output of a app.bsky.contact.dismissMatch call. +type ContactDismissMatch_Output struct { +} + +// ContactDismissMatch calls the XRPC method "app.bsky.contact.dismissMatch". +func ContactDismissMatch(ctx context.Context, c lexutil.LexClient, input *ContactDismissMatch_Input) (*ContactDismissMatch_Output, error) { + var out ContactDismissMatch_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.contact.dismissMatch", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/contactgetMatches.go b/api/bsky/contactgetMatches.go new file mode 100644 index 000000000..c7c0603d9 --- /dev/null +++ b/api/bsky/contactgetMatches.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.getMatches + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ContactGetMatches_Output is the output of a app.bsky.contact.getMatches call. +type ContactGetMatches_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Matches []*ActorDefs_ProfileView `json:"matches" cborgen:"matches"` +} + +// ContactGetMatches calls the XRPC method "app.bsky.contact.getMatches". +func ContactGetMatches(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*ContactGetMatches_Output, error) { + var out ContactGetMatches_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.contact.getMatches", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/contactgetSyncStatus.go b/api/bsky/contactgetSyncStatus.go new file mode 100644 index 000000000..2e157ef15 --- /dev/null +++ b/api/bsky/contactgetSyncStatus.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.getSyncStatus + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ContactGetSyncStatus_Output is the output of a app.bsky.contact.getSyncStatus call. +type ContactGetSyncStatus_Output struct { + // syncStatus: If present, indicates the user has imported their contacts. If not present, indicates the user never used the feature or called `app.bsky.contact.removeData` and didn't import again since. + SyncStatus *ContactDefs_SyncStatus `json:"syncStatus,omitempty" cborgen:"syncStatus,omitempty"` +} + +// ContactGetSyncStatus calls the XRPC method "app.bsky.contact.getSyncStatus". +func ContactGetSyncStatus(ctx context.Context, c lexutil.LexClient) (*ContactGetSyncStatus_Output, error) { + var out ContactGetSyncStatus_Output + + params := map[string]interface{}{} + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.contact.getSyncStatus", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/contactimportContacts.go b/api/bsky/contactimportContacts.go new file mode 100644 index 000000000..46defdb54 --- /dev/null +++ b/api/bsky/contactimportContacts.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.importContacts + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ContactImportContacts_Input is the input argument to a app.bsky.contact.importContacts call. +type ContactImportContacts_Input struct { + // contacts: List of phone numbers in global E.164 format (e.g., '+12125550123'). Phone numbers that cannot be normalized into a valid phone number will be discarded. Should not repeat the 'phone' input used in `app.bsky.contact.verifyPhone`. + Contacts []string `json:"contacts" cborgen:"contacts"` + // token: JWT to authenticate the call. Use the JWT received as a response to the call to `app.bsky.contact.verifyPhone`. + Token string `json:"token" cborgen:"token"` +} + +// ContactImportContacts_Output is the output of a app.bsky.contact.importContacts call. +type ContactImportContacts_Output struct { + // matchesAndContactIndexes: The users that matched during import and their indexes on the input contacts, so the client can correlate with its local list. + MatchesAndContactIndexes []*ContactDefs_MatchAndContactIndex `json:"matchesAndContactIndexes" cborgen:"matchesAndContactIndexes"` +} + +// ContactImportContacts calls the XRPC method "app.bsky.contact.importContacts". +func ContactImportContacts(ctx context.Context, c lexutil.LexClient, input *ContactImportContacts_Input) (*ContactImportContacts_Output, error) { + var out ContactImportContacts_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.contact.importContacts", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/contactremoveData.go b/api/bsky/contactremoveData.go new file mode 100644 index 000000000..36fc2c34e --- /dev/null +++ b/api/bsky/contactremoveData.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.removeData + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ContactRemoveData_Input is the input argument to a app.bsky.contact.removeData call. +type ContactRemoveData_Input struct { +} + +// ContactRemoveData_Output is the output of a app.bsky.contact.removeData call. +type ContactRemoveData_Output struct { +} + +// ContactRemoveData calls the XRPC method "app.bsky.contact.removeData". +func ContactRemoveData(ctx context.Context, c lexutil.LexClient, input *ContactRemoveData_Input) (*ContactRemoveData_Output, error) { + var out ContactRemoveData_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.contact.removeData", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/contactsendNotification.go b/api/bsky/contactsendNotification.go new file mode 100644 index 000000000..3cfd67581 --- /dev/null +++ b/api/bsky/contactsendNotification.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.sendNotification + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ContactSendNotification_Input is the input argument to a app.bsky.contact.sendNotification call. +type ContactSendNotification_Input struct { + // from: The DID of who this notification comes from. + From string `json:"from" cborgen:"from"` + // to: The DID of who this notification should go to. + To string `json:"to" cborgen:"to"` +} + +// ContactSendNotification_Output is the output of a app.bsky.contact.sendNotification call. +type ContactSendNotification_Output struct { +} + +// ContactSendNotification calls the XRPC method "app.bsky.contact.sendNotification". +func ContactSendNotification(ctx context.Context, c lexutil.LexClient, input *ContactSendNotification_Input) (*ContactSendNotification_Output, error) { + var out ContactSendNotification_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.contact.sendNotification", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/contactstartPhoneVerification.go b/api/bsky/contactstartPhoneVerification.go new file mode 100644 index 000000000..8f389186b --- /dev/null +++ b/api/bsky/contactstartPhoneVerification.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.startPhoneVerification + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ContactStartPhoneVerification_Input is the input argument to a app.bsky.contact.startPhoneVerification call. +type ContactStartPhoneVerification_Input struct { + // phone: The phone number to receive the code via SMS. + Phone string `json:"phone" cborgen:"phone"` +} + +// ContactStartPhoneVerification_Output is the output of a app.bsky.contact.startPhoneVerification call. +type ContactStartPhoneVerification_Output struct { +} + +// ContactStartPhoneVerification calls the XRPC method "app.bsky.contact.startPhoneVerification". +func ContactStartPhoneVerification(ctx context.Context, c lexutil.LexClient, input *ContactStartPhoneVerification_Input) (*ContactStartPhoneVerification_Output, error) { + var out ContactStartPhoneVerification_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.contact.startPhoneVerification", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/contactverifyPhone.go b/api/bsky/contactverifyPhone.go new file mode 100644 index 000000000..e05f28875 --- /dev/null +++ b/api/bsky/contactverifyPhone.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.contact.verifyPhone + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ContactVerifyPhone_Input is the input argument to a app.bsky.contact.verifyPhone call. +type ContactVerifyPhone_Input struct { + // code: The code received via SMS as a result of the call to `app.bsky.contact.startPhoneVerification`. + Code string `json:"code" cborgen:"code"` + // phone: The phone number to verify. Should be the same as the one passed to `app.bsky.contact.startPhoneVerification`. + Phone string `json:"phone" cborgen:"phone"` +} + +// ContactVerifyPhone_Output is the output of a app.bsky.contact.verifyPhone call. +type ContactVerifyPhone_Output struct { + // token: JWT to be used in a call to `app.bsky.contact.importContacts`. It is only valid for a single call. + Token string `json:"token" cborgen:"token"` +} + +// ContactVerifyPhone calls the XRPC method "app.bsky.contact.verifyPhone". +func ContactVerifyPhone(ctx context.Context, c lexutil.LexClient, input *ContactVerifyPhone_Input) (*ContactVerifyPhone_Output, error) { + var out ContactVerifyPhone_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.contact.verifyPhone", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/draftcreateDraft.go b/api/bsky/draftcreateDraft.go new file mode 100644 index 000000000..492372d94 --- /dev/null +++ b/api/bsky/draftcreateDraft.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.draft.createDraft + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// DraftCreateDraft_Input is the input argument to a app.bsky.draft.createDraft call. +type DraftCreateDraft_Input struct { + Draft *DraftDefs_Draft `json:"draft" cborgen:"draft"` +} + +// DraftCreateDraft_Output is the output of a app.bsky.draft.createDraft call. +type DraftCreateDraft_Output struct { + // id: The ID of the created draft. + Id string `json:"id" cborgen:"id"` +} + +// DraftCreateDraft calls the XRPC method "app.bsky.draft.createDraft". +func DraftCreateDraft(ctx context.Context, c lexutil.LexClient, input *DraftCreateDraft_Input) (*DraftCreateDraft_Output, error) { + var out DraftCreateDraft_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.draft.createDraft", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/draftdefs.go b/api/bsky/draftdefs.go new file mode 100644 index 000000000..5d45e9c8b --- /dev/null +++ b/api/bsky/draftdefs.go @@ -0,0 +1,204 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.draft.defs + +package bsky + +import ( + "encoding/json" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// DraftDefs_Draft is a "draft" in the app.bsky.draft.defs schema. +// +// A draft containing an array of draft posts. +type DraftDefs_Draft struct { + // langs: Indicates human language of posts primary text content. + Langs []string `json:"langs,omitempty" cborgen:"langs,omitempty"` + // postgateEmbeddingRules: Embedding rules for the postgates to be created when this draft is published. + PostgateEmbeddingRules []*DraftDefs_Draft_PostgateEmbeddingRules_Elem `json:"postgateEmbeddingRules,omitempty" cborgen:"postgateEmbeddingRules,omitempty"` + // posts: Array of draft posts that compose this draft. + Posts []*DraftDefs_DraftPost `json:"posts" cborgen:"posts"` + // threadgateAllow: Allow-rules for the threadgate to be created when this draft is published. + ThreadgateAllow []*DraftDefs_Draft_ThreadgateAllow_Elem `json:"threadgateAllow,omitempty" cborgen:"threadgateAllow,omitempty"` +} + +// DraftDefs_DraftEmbedCaption is a "draftEmbedCaption" in the app.bsky.draft.defs schema. +type DraftDefs_DraftEmbedCaption struct { + Content string `json:"content" cborgen:"content"` + Lang string `json:"lang" cborgen:"lang"` +} + +// DraftDefs_DraftEmbedExternal is a "draftEmbedExternal" in the app.bsky.draft.defs schema. +type DraftDefs_DraftEmbedExternal struct { + Uri string `json:"uri" cborgen:"uri"` +} + +// DraftDefs_DraftEmbedImage is a "draftEmbedImage" in the app.bsky.draft.defs schema. +type DraftDefs_DraftEmbedImage struct { + Alt *string `json:"alt,omitempty" cborgen:"alt,omitempty"` + LocalRef *DraftDefs_DraftEmbedLocalRef `json:"localRef" cborgen:"localRef"` +} + +// DraftDefs_DraftEmbedLocalRef is a "draftEmbedLocalRef" in the app.bsky.draft.defs schema. +type DraftDefs_DraftEmbedLocalRef struct { + // path: Local, on-device ref to file to be embedded. Embeds are currently device-bound for drafts. + Path string `json:"path" cborgen:"path"` +} + +// DraftDefs_DraftEmbedRecord is a "draftEmbedRecord" in the app.bsky.draft.defs schema. +type DraftDefs_DraftEmbedRecord struct { + Record *comatproto.RepoStrongRef `json:"record" cborgen:"record"` +} + +// DraftDefs_DraftEmbedVideo is a "draftEmbedVideo" in the app.bsky.draft.defs schema. +type DraftDefs_DraftEmbedVideo struct { + Alt *string `json:"alt,omitempty" cborgen:"alt,omitempty"` + Captions []*DraftDefs_DraftEmbedCaption `json:"captions,omitempty" cborgen:"captions,omitempty"` + LocalRef *DraftDefs_DraftEmbedLocalRef `json:"localRef" cborgen:"localRef"` +} + +// DraftDefs_DraftPost is a "draftPost" in the app.bsky.draft.defs schema. +// +// One of the posts that compose a draft. +type DraftDefs_DraftPost struct { + EmbedExternals []*DraftDefs_DraftEmbedExternal `json:"embedExternals,omitempty" cborgen:"embedExternals,omitempty"` + EmbedImages []*DraftDefs_DraftEmbedImage `json:"embedImages,omitempty" cborgen:"embedImages,omitempty"` + EmbedRecords []*DraftDefs_DraftEmbedRecord `json:"embedRecords,omitempty" cborgen:"embedRecords,omitempty"` + EmbedVideos []*DraftDefs_DraftEmbedVideo `json:"embedVideos,omitempty" cborgen:"embedVideos,omitempty"` + // labels: Self-label values for this post. Effectively content warnings. + Labels *DraftDefs_DraftPost_Labels `json:"labels,omitempty" cborgen:"labels,omitempty"` + // text: The primary post content. + Text string `json:"text" cborgen:"text"` +} + +// Self-label values for this post. Effectively content warnings. +type DraftDefs_DraftPost_Labels struct { + LabelDefs_SelfLabels *comatproto.LabelDefs_SelfLabels +} + +func (t *DraftDefs_DraftPost_Labels) MarshalJSON() ([]byte, error) { + if t.LabelDefs_SelfLabels != nil { + t.LabelDefs_SelfLabels.LexiconTypeID = "com.atproto.label.defs#selfLabels" + return json.Marshal(t.LabelDefs_SelfLabels) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *DraftDefs_DraftPost_Labels) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return json.Unmarshal(b, t.LabelDefs_SelfLabels) + default: + return nil + } +} + +// DraftDefs_DraftView is a "draftView" in the app.bsky.draft.defs schema. +// +// View to present drafts data to users. +type DraftDefs_DraftView struct { + // createdAt: The time the draft was created. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Draft *DraftDefs_Draft `json:"draft" cborgen:"draft"` + // id: A TID to be used as a draft identifier. + Id string `json:"id" cborgen:"id"` + // updatedAt: The time the draft was last updated. + UpdatedAt string `json:"updatedAt" cborgen:"updatedAt"` +} + +// DraftDefs_DraftWithId is a "draftWithId" in the app.bsky.draft.defs schema. +// +// A draft with an identifier, used to store drafts in private storage (stash). +type DraftDefs_DraftWithId struct { + Draft *DraftDefs_Draft `json:"draft" cborgen:"draft"` + // id: A TID to be used as a draft identifier. + Id string `json:"id" cborgen:"id"` +} + +type DraftDefs_Draft_PostgateEmbeddingRules_Elem struct { + FeedPostgate_DisableRule *FeedPostgate_DisableRule +} + +func (t *DraftDefs_Draft_PostgateEmbeddingRules_Elem) MarshalJSON() ([]byte, error) { + if t.FeedPostgate_DisableRule != nil { + t.FeedPostgate_DisableRule.LexiconTypeID = "app.bsky.feed.postgate#disableRule" + return json.Marshal(t.FeedPostgate_DisableRule) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *DraftDefs_Draft_PostgateEmbeddingRules_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.postgate#disableRule": + t.FeedPostgate_DisableRule = new(FeedPostgate_DisableRule) + return json.Unmarshal(b, t.FeedPostgate_DisableRule) + default: + return nil + } +} + +type DraftDefs_Draft_ThreadgateAllow_Elem struct { + FeedThreadgate_MentionRule *FeedThreadgate_MentionRule + FeedThreadgate_FollowerRule *FeedThreadgate_FollowerRule + FeedThreadgate_FollowingRule *FeedThreadgate_FollowingRule + FeedThreadgate_ListRule *FeedThreadgate_ListRule +} + +func (t *DraftDefs_Draft_ThreadgateAllow_Elem) MarshalJSON() ([]byte, error) { + if t.FeedThreadgate_MentionRule != nil { + t.FeedThreadgate_MentionRule.LexiconTypeID = "app.bsky.feed.threadgate#mentionRule" + return json.Marshal(t.FeedThreadgate_MentionRule) + } + if t.FeedThreadgate_FollowerRule != nil { + t.FeedThreadgate_FollowerRule.LexiconTypeID = "app.bsky.feed.threadgate#followerRule" + return json.Marshal(t.FeedThreadgate_FollowerRule) + } + if t.FeedThreadgate_FollowingRule != nil { + t.FeedThreadgate_FollowingRule.LexiconTypeID = "app.bsky.feed.threadgate#followingRule" + return json.Marshal(t.FeedThreadgate_FollowingRule) + } + if t.FeedThreadgate_ListRule != nil { + t.FeedThreadgate_ListRule.LexiconTypeID = "app.bsky.feed.threadgate#listRule" + return json.Marshal(t.FeedThreadgate_ListRule) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *DraftDefs_Draft_ThreadgateAllow_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.threadgate#mentionRule": + t.FeedThreadgate_MentionRule = new(FeedThreadgate_MentionRule) + return json.Unmarshal(b, t.FeedThreadgate_MentionRule) + case "app.bsky.feed.threadgate#followerRule": + t.FeedThreadgate_FollowerRule = new(FeedThreadgate_FollowerRule) + return json.Unmarshal(b, t.FeedThreadgate_FollowerRule) + case "app.bsky.feed.threadgate#followingRule": + t.FeedThreadgate_FollowingRule = new(FeedThreadgate_FollowingRule) + return json.Unmarshal(b, t.FeedThreadgate_FollowingRule) + case "app.bsky.feed.threadgate#listRule": + t.FeedThreadgate_ListRule = new(FeedThreadgate_ListRule) + return json.Unmarshal(b, t.FeedThreadgate_ListRule) + default: + return nil + } +} diff --git a/api/bsky/draftdeleteDraft.go b/api/bsky/draftdeleteDraft.go new file mode 100644 index 000000000..dafd61f0f --- /dev/null +++ b/api/bsky/draftdeleteDraft.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.draft.deleteDraft + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// DraftDeleteDraft_Input is the input argument to a app.bsky.draft.deleteDraft call. +type DraftDeleteDraft_Input struct { + Id string `json:"id" cborgen:"id"` +} + +// DraftDeleteDraft calls the XRPC method "app.bsky.draft.deleteDraft". +func DraftDeleteDraft(ctx context.Context, c lexutil.LexClient, input *DraftDeleteDraft_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.draft.deleteDraft", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/draftgetDrafts.go b/api/bsky/draftgetDrafts.go new file mode 100644 index 000000000..64b80ac11 --- /dev/null +++ b/api/bsky/draftgetDrafts.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.draft.getDrafts + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// DraftGetDrafts_Output is the output of a app.bsky.draft.getDrafts call. +type DraftGetDrafts_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Drafts []*DraftDefs_DraftView `json:"drafts" cborgen:"drafts"` +} + +// DraftGetDrafts calls the XRPC method "app.bsky.draft.getDrafts". +func DraftGetDrafts(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*DraftGetDrafts_Output, error) { + var out DraftGetDrafts_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.draft.getDrafts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/draftupdateDraft.go b/api/bsky/draftupdateDraft.go new file mode 100644 index 000000000..c0a25829a --- /dev/null +++ b/api/bsky/draftupdateDraft.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.draft.updateDraft + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// DraftUpdateDraft_Input is the input argument to a app.bsky.draft.updateDraft call. +type DraftUpdateDraft_Input struct { + Draft *DraftDefs_DraftWithId `json:"draft" cborgen:"draft"` +} + +// DraftUpdateDraft calls the XRPC method "app.bsky.draft.updateDraft". +func DraftUpdateDraft(ctx context.Context, c lexutil.LexClient, input *DraftUpdateDraft_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.draft.updateDraft", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/embeddefs.go b/api/bsky/embeddefs.go new file mode 100644 index 000000000..449b629f6 --- /dev/null +++ b/api/bsky/embeddefs.go @@ -0,0 +1,13 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.embed.defs + +package bsky + +// EmbedDefs_AspectRatio is a "aspectRatio" in the app.bsky.embed.defs schema. +// +// width:height represents an aspect ratio. It may be approximate, and may not correspond to absolute dimensions in any given unit. +type EmbedDefs_AspectRatio struct { + Height int64 `json:"height" cborgen:"height"` + Width int64 `json:"width" cborgen:"width"` +} diff --git a/api/bsky/embedexternal.go b/api/bsky/embedexternal.go index 4d30c8629..e894be5bc 100644 --- a/api/bsky/embedexternal.go +++ b/api/bsky/embedexternal.go @@ -1,36 +1,43 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.embed.external + +package bsky import ( - "github.com/bluesky-social/indigo/lex/util" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.embed.external - func init() { + lexutil.RegisterType("app.bsky.embed.external#main", &EmbedExternal{}) } +// EmbedExternal is a "main" in the app.bsky.embed.external schema. +// +// A representation of some externally linked content (eg, a URL and 'card'), embedded in a Bluesky record (eg, a post). type EmbedExternal struct { - LexiconTypeID string `json:"$type,omitempty"` + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.external"` External *EmbedExternal_External `json:"external" cborgen:"external"` } +// EmbedExternal_External is a "external" in the app.bsky.embed.external schema. type EmbedExternal_External struct { - LexiconTypeID string `json:"$type,omitempty"` - Description string `json:"description" cborgen:"description"` - Thumb *util.Blob `json:"thumb,omitempty" cborgen:"thumb"` - Title string `json:"title" cborgen:"title"` - Uri string `json:"uri" cborgen:"uri"` + Description string `json:"description" cborgen:"description"` + Thumb *lexutil.LexBlob `json:"thumb,omitempty" cborgen:"thumb,omitempty"` + Title string `json:"title" cborgen:"title"` + Uri string `json:"uri" cborgen:"uri"` } -type EmbedExternal_Presented struct { - LexiconTypeID string `json:"$type,omitempty"` - External *EmbedExternal_PresentedExternal `json:"external" cborgen:"external"` +// EmbedExternal_View is a "view" in the app.bsky.embed.external schema. +type EmbedExternal_View struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.external#view"` + External *EmbedExternal_ViewExternal `json:"external" cborgen:"external"` } -type EmbedExternal_PresentedExternal struct { - LexiconTypeID string `json:"$type,omitempty"` - Description string `json:"description" cborgen:"description"` - Thumb *string `json:"thumb,omitempty" cborgen:"thumb"` - Title string `json:"title" cborgen:"title"` - Uri string `json:"uri" cborgen:"uri"` +// EmbedExternal_ViewExternal is a "viewExternal" in the app.bsky.embed.external schema. +type EmbedExternal_ViewExternal struct { + Description string `json:"description" cborgen:"description"` + Thumb *string `json:"thumb,omitempty" cborgen:"thumb,omitempty"` + Title string `json:"title" cborgen:"title"` + Uri string `json:"uri" cborgen:"uri"` } diff --git a/api/bsky/embedimages.go b/api/bsky/embedimages.go index 0e307f1a6..ebc98512f 100644 --- a/api/bsky/embedimages.go +++ b/api/bsky/embedimages.go @@ -1,33 +1,44 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.embed.images + +package bsky import ( - "github.com/bluesky-social/indigo/lex/util" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.embed.images - func init() { + lexutil.RegisterType("app.bsky.embed.images#main", &EmbedImages{}) } +// EmbedImages is a "main" in the app.bsky.embed.images schema. type EmbedImages struct { - LexiconTypeID string `json:"$type,omitempty"` + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.images"` Images []*EmbedImages_Image `json:"images" cborgen:"images"` } +// EmbedImages_Image is a "image" in the app.bsky.embed.images schema. type EmbedImages_Image struct { - LexiconTypeID string `json:"$type,omitempty"` - Alt string `json:"alt" cborgen:"alt"` - Image *util.Blob `json:"image" cborgen:"image"` + // alt: Alt text description of the image, for accessibility. + Alt string `json:"alt" cborgen:"alt"` + AspectRatio *EmbedDefs_AspectRatio `json:"aspectRatio,omitempty" cborgen:"aspectRatio,omitempty"` + Image *lexutil.LexBlob `json:"image" cborgen:"image"` } -type EmbedImages_Presented struct { - LexiconTypeID string `json:"$type,omitempty"` - Images []*EmbedImages_PresentedImage `json:"images" cborgen:"images"` +// EmbedImages_View is a "view" in the app.bsky.embed.images schema. +type EmbedImages_View struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.images#view"` + Images []*EmbedImages_ViewImage `json:"images" cborgen:"images"` } -type EmbedImages_PresentedImage struct { - LexiconTypeID string `json:"$type,omitempty"` - Alt string `json:"alt" cborgen:"alt"` - Fullsize string `json:"fullsize" cborgen:"fullsize"` - Thumb string `json:"thumb" cborgen:"thumb"` +// EmbedImages_ViewImage is a "viewImage" in the app.bsky.embed.images schema. +type EmbedImages_ViewImage struct { + // alt: Alt text description of the image, for accessibility. + Alt string `json:"alt" cborgen:"alt"` + AspectRatio *EmbedDefs_AspectRatio `json:"aspectRatio,omitempty" cborgen:"aspectRatio,omitempty"` + // fullsize: Fully-qualified URL where a large version of the image can be fetched. May or may not be the exact original blob. For example, CDN location provided by the App View. + Fullsize string `json:"fullsize" cborgen:"fullsize"` + // thumb: Fully-qualified URL where a thumbnail of the image can be fetched. For example, CDN location provided by the App View. + Thumb string `json:"thumb" cborgen:"thumb"` } diff --git a/api/bsky/embedrecord.go b/api/bsky/embedrecord.go new file mode 100644 index 000000000..4a7d6f433 --- /dev/null +++ b/api/bsky/embedrecord.go @@ -0,0 +1,210 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.embed.record + +package bsky + +import ( + "encoding/json" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.embed.record#main", &EmbedRecord{}) +} + +// EmbedRecord is a "main" in the app.bsky.embed.record schema. +type EmbedRecord struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.record"` + Record *comatproto.RepoStrongRef `json:"record" cborgen:"record"` +} + +// EmbedRecord_View is a "view" in the app.bsky.embed.record schema. +type EmbedRecord_View struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.record#view"` + Record *EmbedRecord_View_Record `json:"record" cborgen:"record"` +} + +// EmbedRecord_ViewBlocked is a "viewBlocked" in the app.bsky.embed.record schema. +type EmbedRecord_ViewBlocked struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.record#viewBlocked"` + Author *FeedDefs_BlockedAuthor `json:"author" cborgen:"author"` + Blocked bool `json:"blocked" cborgen:"blocked"` + Uri string `json:"uri" cborgen:"uri"` +} + +// EmbedRecord_ViewDetached is a "viewDetached" in the app.bsky.embed.record schema. +type EmbedRecord_ViewDetached struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.record#viewDetached"` + Detached bool `json:"detached" cborgen:"detached"` + Uri string `json:"uri" cborgen:"uri"` +} + +// EmbedRecord_ViewNotFound is a "viewNotFound" in the app.bsky.embed.record schema. +type EmbedRecord_ViewNotFound struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.record#viewNotFound"` + NotFound bool `json:"notFound" cborgen:"notFound"` + Uri string `json:"uri" cborgen:"uri"` +} + +// EmbedRecord_ViewRecord is a "viewRecord" in the app.bsky.embed.record schema. +type EmbedRecord_ViewRecord struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.record#viewRecord"` + Author *ActorDefs_ProfileViewBasic `json:"author" cborgen:"author"` + Cid string `json:"cid" cborgen:"cid"` + Embeds []*EmbedRecord_ViewRecord_Embeds_Elem `json:"embeds,omitempty" cborgen:"embeds,omitempty"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + LikeCount *int64 `json:"likeCount,omitempty" cborgen:"likeCount,omitempty"` + QuoteCount *int64 `json:"quoteCount,omitempty" cborgen:"quoteCount,omitempty"` + ReplyCount *int64 `json:"replyCount,omitempty" cborgen:"replyCount,omitempty"` + RepostCount *int64 `json:"repostCount,omitempty" cborgen:"repostCount,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + // value: The record data itself. + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` +} + +type EmbedRecord_ViewRecord_Embeds_Elem struct { + EmbedImages_View *EmbedImages_View + EmbedVideo_View *EmbedVideo_View + EmbedExternal_View *EmbedExternal_View + EmbedRecord_View *EmbedRecord_View + EmbedRecordWithMedia_View *EmbedRecordWithMedia_View +} + +func (t *EmbedRecord_ViewRecord_Embeds_Elem) MarshalJSON() ([]byte, error) { + if t.EmbedImages_View != nil { + t.EmbedImages_View.LexiconTypeID = "app.bsky.embed.images#view" + return json.Marshal(t.EmbedImages_View) + } + if t.EmbedVideo_View != nil { + t.EmbedVideo_View.LexiconTypeID = "app.bsky.embed.video#view" + return json.Marshal(t.EmbedVideo_View) + } + if t.EmbedExternal_View != nil { + t.EmbedExternal_View.LexiconTypeID = "app.bsky.embed.external#view" + return json.Marshal(t.EmbedExternal_View) + } + if t.EmbedRecord_View != nil { + t.EmbedRecord_View.LexiconTypeID = "app.bsky.embed.record#view" + return json.Marshal(t.EmbedRecord_View) + } + if t.EmbedRecordWithMedia_View != nil { + t.EmbedRecordWithMedia_View.LexiconTypeID = "app.bsky.embed.recordWithMedia#view" + return json.Marshal(t.EmbedRecordWithMedia_View) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *EmbedRecord_ViewRecord_Embeds_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.images#view": + t.EmbedImages_View = new(EmbedImages_View) + return json.Unmarshal(b, t.EmbedImages_View) + case "app.bsky.embed.video#view": + t.EmbedVideo_View = new(EmbedVideo_View) + return json.Unmarshal(b, t.EmbedVideo_View) + case "app.bsky.embed.external#view": + t.EmbedExternal_View = new(EmbedExternal_View) + return json.Unmarshal(b, t.EmbedExternal_View) + case "app.bsky.embed.record#view": + t.EmbedRecord_View = new(EmbedRecord_View) + return json.Unmarshal(b, t.EmbedRecord_View) + case "app.bsky.embed.recordWithMedia#view": + t.EmbedRecordWithMedia_View = new(EmbedRecordWithMedia_View) + return json.Unmarshal(b, t.EmbedRecordWithMedia_View) + default: + return nil + } +} + +type EmbedRecord_View_Record struct { + EmbedRecord_ViewRecord *EmbedRecord_ViewRecord + EmbedRecord_ViewNotFound *EmbedRecord_ViewNotFound + EmbedRecord_ViewBlocked *EmbedRecord_ViewBlocked + EmbedRecord_ViewDetached *EmbedRecord_ViewDetached + FeedDefs_GeneratorView *FeedDefs_GeneratorView + GraphDefs_ListView *GraphDefs_ListView + LabelerDefs_LabelerView *LabelerDefs_LabelerView + GraphDefs_StarterPackViewBasic *GraphDefs_StarterPackViewBasic +} + +func (t *EmbedRecord_View_Record) MarshalJSON() ([]byte, error) { + if t.EmbedRecord_ViewRecord != nil { + t.EmbedRecord_ViewRecord.LexiconTypeID = "app.bsky.embed.record#viewRecord" + return json.Marshal(t.EmbedRecord_ViewRecord) + } + if t.EmbedRecord_ViewNotFound != nil { + t.EmbedRecord_ViewNotFound.LexiconTypeID = "app.bsky.embed.record#viewNotFound" + return json.Marshal(t.EmbedRecord_ViewNotFound) + } + if t.EmbedRecord_ViewBlocked != nil { + t.EmbedRecord_ViewBlocked.LexiconTypeID = "app.bsky.embed.record#viewBlocked" + return json.Marshal(t.EmbedRecord_ViewBlocked) + } + if t.EmbedRecord_ViewDetached != nil { + t.EmbedRecord_ViewDetached.LexiconTypeID = "app.bsky.embed.record#viewDetached" + return json.Marshal(t.EmbedRecord_ViewDetached) + } + if t.FeedDefs_GeneratorView != nil { + t.FeedDefs_GeneratorView.LexiconTypeID = "app.bsky.feed.defs#generatorView" + return json.Marshal(t.FeedDefs_GeneratorView) + } + if t.GraphDefs_ListView != nil { + t.GraphDefs_ListView.LexiconTypeID = "app.bsky.graph.defs#listView" + return json.Marshal(t.GraphDefs_ListView) + } + if t.LabelerDefs_LabelerView != nil { + t.LabelerDefs_LabelerView.LexiconTypeID = "app.bsky.labeler.defs#labelerView" + return json.Marshal(t.LabelerDefs_LabelerView) + } + if t.GraphDefs_StarterPackViewBasic != nil { + t.GraphDefs_StarterPackViewBasic.LexiconTypeID = "app.bsky.graph.defs#starterPackViewBasic" + return json.Marshal(t.GraphDefs_StarterPackViewBasic) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *EmbedRecord_View_Record) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.record#viewRecord": + t.EmbedRecord_ViewRecord = new(EmbedRecord_ViewRecord) + return json.Unmarshal(b, t.EmbedRecord_ViewRecord) + case "app.bsky.embed.record#viewNotFound": + t.EmbedRecord_ViewNotFound = new(EmbedRecord_ViewNotFound) + return json.Unmarshal(b, t.EmbedRecord_ViewNotFound) + case "app.bsky.embed.record#viewBlocked": + t.EmbedRecord_ViewBlocked = new(EmbedRecord_ViewBlocked) + return json.Unmarshal(b, t.EmbedRecord_ViewBlocked) + case "app.bsky.embed.record#viewDetached": + t.EmbedRecord_ViewDetached = new(EmbedRecord_ViewDetached) + return json.Unmarshal(b, t.EmbedRecord_ViewDetached) + case "app.bsky.feed.defs#generatorView": + t.FeedDefs_GeneratorView = new(FeedDefs_GeneratorView) + return json.Unmarshal(b, t.FeedDefs_GeneratorView) + case "app.bsky.graph.defs#listView": + t.GraphDefs_ListView = new(GraphDefs_ListView) + return json.Unmarshal(b, t.GraphDefs_ListView) + case "app.bsky.labeler.defs#labelerView": + t.LabelerDefs_LabelerView = new(LabelerDefs_LabelerView) + return json.Unmarshal(b, t.LabelerDefs_LabelerView) + case "app.bsky.graph.defs#starterPackViewBasic": + t.GraphDefs_StarterPackViewBasic = new(GraphDefs_StarterPackViewBasic) + return json.Unmarshal(b, t.GraphDefs_StarterPackViewBasic) + default: + return nil + } +} diff --git a/api/bsky/embedrecordWithMedia.go b/api/bsky/embedrecordWithMedia.go new file mode 100644 index 000000000..74157b1e5 --- /dev/null +++ b/api/bsky/embedrecordWithMedia.go @@ -0,0 +1,158 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.embed.recordWithMedia + +package bsky + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func init() { + lexutil.RegisterType("app.bsky.embed.recordWithMedia#main", &EmbedRecordWithMedia{}) +} + +// EmbedRecordWithMedia is a "main" in the app.bsky.embed.recordWithMedia schema. +type EmbedRecordWithMedia struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.recordWithMedia"` + Media *EmbedRecordWithMedia_Media `json:"media" cborgen:"media"` + Record *EmbedRecord `json:"record" cborgen:"record"` +} + +type EmbedRecordWithMedia_Media struct { + EmbedImages *EmbedImages + EmbedVideo *EmbedVideo + EmbedExternal *EmbedExternal +} + +func (t *EmbedRecordWithMedia_Media) MarshalJSON() ([]byte, error) { + if t.EmbedImages != nil { + t.EmbedImages.LexiconTypeID = "app.bsky.embed.images" + return json.Marshal(t.EmbedImages) + } + if t.EmbedVideo != nil { + t.EmbedVideo.LexiconTypeID = "app.bsky.embed.video" + return json.Marshal(t.EmbedVideo) + } + if t.EmbedExternal != nil { + t.EmbedExternal.LexiconTypeID = "app.bsky.embed.external" + return json.Marshal(t.EmbedExternal) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *EmbedRecordWithMedia_Media) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.images": + t.EmbedImages = new(EmbedImages) + return json.Unmarshal(b, t.EmbedImages) + case "app.bsky.embed.video": + t.EmbedVideo = new(EmbedVideo) + return json.Unmarshal(b, t.EmbedVideo) + case "app.bsky.embed.external": + t.EmbedExternal = new(EmbedExternal) + return json.Unmarshal(b, t.EmbedExternal) + default: + return nil + } +} + +func (t *EmbedRecordWithMedia_Media) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.EmbedImages != nil { + return t.EmbedImages.MarshalCBOR(w) + } + if t.EmbedVideo != nil { + return t.EmbedVideo.MarshalCBOR(w) + } + if t.EmbedExternal != nil { + return t.EmbedExternal.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *EmbedRecordWithMedia_Media) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.images": + t.EmbedImages = new(EmbedImages) + return t.EmbedImages.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.embed.video": + t.EmbedVideo = new(EmbedVideo) + return t.EmbedVideo.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.embed.external": + t.EmbedExternal = new(EmbedExternal) + return t.EmbedExternal.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } +} + +// EmbedRecordWithMedia_View is a "view" in the app.bsky.embed.recordWithMedia schema. +type EmbedRecordWithMedia_View struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.recordWithMedia#view"` + Media *EmbedRecordWithMedia_View_Media `json:"media" cborgen:"media"` + Record *EmbedRecord_View `json:"record" cborgen:"record"` +} + +type EmbedRecordWithMedia_View_Media struct { + EmbedImages_View *EmbedImages_View + EmbedVideo_View *EmbedVideo_View + EmbedExternal_View *EmbedExternal_View +} + +func (t *EmbedRecordWithMedia_View_Media) MarshalJSON() ([]byte, error) { + if t.EmbedImages_View != nil { + t.EmbedImages_View.LexiconTypeID = "app.bsky.embed.images#view" + return json.Marshal(t.EmbedImages_View) + } + if t.EmbedVideo_View != nil { + t.EmbedVideo_View.LexiconTypeID = "app.bsky.embed.video#view" + return json.Marshal(t.EmbedVideo_View) + } + if t.EmbedExternal_View != nil { + t.EmbedExternal_View.LexiconTypeID = "app.bsky.embed.external#view" + return json.Marshal(t.EmbedExternal_View) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *EmbedRecordWithMedia_View_Media) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.images#view": + t.EmbedImages_View = new(EmbedImages_View) + return json.Unmarshal(b, t.EmbedImages_View) + case "app.bsky.embed.video#view": + t.EmbedVideo_View = new(EmbedVideo_View) + return json.Unmarshal(b, t.EmbedVideo_View) + case "app.bsky.embed.external#view": + t.EmbedExternal_View = new(EmbedExternal_View) + return json.Unmarshal(b, t.EmbedExternal_View) + default: + return nil + } +} diff --git a/api/bsky/embedvideo.go b/api/bsky/embedvideo.go new file mode 100644 index 000000000..77af759dc --- /dev/null +++ b/api/bsky/embedvideo.go @@ -0,0 +1,40 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.embed.video + +package bsky + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.embed.video#main", &EmbedVideo{}) +} + +// EmbedVideo is a "main" in the app.bsky.embed.video schema. +type EmbedVideo struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.video"` + // alt: Alt text description of the video, for accessibility. + Alt *string `json:"alt,omitempty" cborgen:"alt,omitempty"` + AspectRatio *EmbedDefs_AspectRatio `json:"aspectRatio,omitempty" cborgen:"aspectRatio,omitempty"` + Captions []*EmbedVideo_Caption `json:"captions,omitempty" cborgen:"captions,omitempty"` + // video: The mp4 video file. May be up to 100mb, formerly limited to 50mb. + Video *lexutil.LexBlob `json:"video" cborgen:"video"` +} + +// EmbedVideo_Caption is a "caption" in the app.bsky.embed.video schema. +type EmbedVideo_Caption struct { + File *lexutil.LexBlob `json:"file" cborgen:"file"` + Lang string `json:"lang" cborgen:"lang"` +} + +// EmbedVideo_View is a "view" in the app.bsky.embed.video schema. +type EmbedVideo_View struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.embed.video#view"` + Alt *string `json:"alt,omitempty" cborgen:"alt,omitempty"` + AspectRatio *EmbedDefs_AspectRatio `json:"aspectRatio,omitempty" cborgen:"aspectRatio,omitempty"` + Cid string `json:"cid" cborgen:"cid"` + Playlist string `json:"playlist" cborgen:"playlist"` + Thumbnail *string `json:"thumbnail,omitempty" cborgen:"thumbnail,omitempty"` +} diff --git a/api/bsky/feeddefs.go b/api/bsky/feeddefs.go new file mode 100644 index 000000000..f1b4eb7b5 --- /dev/null +++ b/api/bsky/feeddefs.go @@ -0,0 +1,479 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.defs + +package bsky + +import ( + "encoding/json" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedDefs_BlockedAuthor is a "blockedAuthor" in the app.bsky.feed.defs schema. +type FeedDefs_BlockedAuthor struct { + Did string `json:"did" cborgen:"did"` + Viewer *ActorDefs_ViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// FeedDefs_BlockedPost is a "blockedPost" in the app.bsky.feed.defs schema. +type FeedDefs_BlockedPost struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#blockedPost"` + Author *FeedDefs_BlockedAuthor `json:"author" cborgen:"author"` + Blocked bool `json:"blocked" cborgen:"blocked"` + Uri string `json:"uri" cborgen:"uri"` +} + +// FeedDefs_FeedViewPost is a "feedViewPost" in the app.bsky.feed.defs schema. +type FeedDefs_FeedViewPost struct { + // feedContext: Context provided by feed generator that may be passed back alongside interactions. + FeedContext *string `json:"feedContext,omitempty" cborgen:"feedContext,omitempty"` + Post *FeedDefs_PostView `json:"post" cborgen:"post"` + Reason *FeedDefs_FeedViewPost_Reason `json:"reason,omitempty" cborgen:"reason,omitempty"` + Reply *FeedDefs_ReplyRef `json:"reply,omitempty" cborgen:"reply,omitempty"` + // reqId: Unique identifier per request that may be passed back alongside interactions. + ReqId *string `json:"reqId,omitempty" cborgen:"reqId,omitempty"` +} + +type FeedDefs_FeedViewPost_Reason struct { + FeedDefs_ReasonRepost *FeedDefs_ReasonRepost + FeedDefs_ReasonPin *FeedDefs_ReasonPin +} + +func (t *FeedDefs_FeedViewPost_Reason) MarshalJSON() ([]byte, error) { + if t.FeedDefs_ReasonRepost != nil { + t.FeedDefs_ReasonRepost.LexiconTypeID = "app.bsky.feed.defs#reasonRepost" + return json.Marshal(t.FeedDefs_ReasonRepost) + } + if t.FeedDefs_ReasonPin != nil { + t.FeedDefs_ReasonPin.LexiconTypeID = "app.bsky.feed.defs#reasonPin" + return json.Marshal(t.FeedDefs_ReasonPin) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedDefs_FeedViewPost_Reason) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.defs#reasonRepost": + t.FeedDefs_ReasonRepost = new(FeedDefs_ReasonRepost) + return json.Unmarshal(b, t.FeedDefs_ReasonRepost) + case "app.bsky.feed.defs#reasonPin": + t.FeedDefs_ReasonPin = new(FeedDefs_ReasonPin) + return json.Unmarshal(b, t.FeedDefs_ReasonPin) + default: + return nil + } +} + +// FeedDefs_GeneratorView is a "generatorView" in the app.bsky.feed.defs schema. +type FeedDefs_GeneratorView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#generatorView"` + AcceptsInteractions *bool `json:"acceptsInteractions,omitempty" cborgen:"acceptsInteractions,omitempty"` + Avatar *string `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + Cid string `json:"cid" cborgen:"cid"` + ContentMode *string `json:"contentMode,omitempty" cborgen:"contentMode,omitempty"` + Creator *ActorDefs_ProfileView `json:"creator" cborgen:"creator"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + DescriptionFacets []*RichtextFacet `json:"descriptionFacets,omitempty" cborgen:"descriptionFacets,omitempty"` + Did string `json:"did" cborgen:"did"` + DisplayName string `json:"displayName" cborgen:"displayName"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + LikeCount *int64 `json:"likeCount,omitempty" cborgen:"likeCount,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + Viewer *FeedDefs_GeneratorViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// FeedDefs_GeneratorViewerState is a "generatorViewerState" in the app.bsky.feed.defs schema. +type FeedDefs_GeneratorViewerState struct { + Like *string `json:"like,omitempty" cborgen:"like,omitempty"` +} + +// FeedDefs_Interaction is a "interaction" in the app.bsky.feed.defs schema. +type FeedDefs_Interaction struct { + Event *string `json:"event,omitempty" cborgen:"event,omitempty"` + // feedContext: Context on a feed item that was originally supplied by the feed generator on getFeedSkeleton. + FeedContext *string `json:"feedContext,omitempty" cborgen:"feedContext,omitempty"` + Item *string `json:"item,omitempty" cborgen:"item,omitempty"` + // reqId: Unique identifier per request that may be passed back alongside interactions. + ReqId *string `json:"reqId,omitempty" cborgen:"reqId,omitempty"` +} + +// FeedDefs_NotFoundPost is a "notFoundPost" in the app.bsky.feed.defs schema. +type FeedDefs_NotFoundPost struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#notFoundPost"` + NotFound bool `json:"notFound" cborgen:"notFound"` + Uri string `json:"uri" cborgen:"uri"` +} + +// FeedDefs_PostView is a "postView" in the app.bsky.feed.defs schema. +type FeedDefs_PostView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#postView"` + Author *ActorDefs_ProfileViewBasic `json:"author" cborgen:"author"` + BookmarkCount *int64 `json:"bookmarkCount,omitempty" cborgen:"bookmarkCount,omitempty"` + Cid string `json:"cid" cborgen:"cid"` + // debug: Debug information for internal development + Debug *interface{} `json:"debug,omitempty" cborgen:"debug,omitempty"` + Embed *FeedDefs_PostView_Embed `json:"embed,omitempty" cborgen:"embed,omitempty"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + LikeCount *int64 `json:"likeCount,omitempty" cborgen:"likeCount,omitempty"` + QuoteCount *int64 `json:"quoteCount,omitempty" cborgen:"quoteCount,omitempty"` + Record *lexutil.LexiconTypeDecoder `json:"record" cborgen:"record"` + ReplyCount *int64 `json:"replyCount,omitempty" cborgen:"replyCount,omitempty"` + RepostCount *int64 `json:"repostCount,omitempty" cborgen:"repostCount,omitempty"` + Threadgate *FeedDefs_ThreadgateView `json:"threadgate,omitempty" cborgen:"threadgate,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + Viewer *FeedDefs_ViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +type FeedDefs_PostView_Embed struct { + EmbedImages_View *EmbedImages_View + EmbedVideo_View *EmbedVideo_View + EmbedExternal_View *EmbedExternal_View + EmbedRecord_View *EmbedRecord_View + EmbedRecordWithMedia_View *EmbedRecordWithMedia_View +} + +func (t *FeedDefs_PostView_Embed) MarshalJSON() ([]byte, error) { + if t.EmbedImages_View != nil { + t.EmbedImages_View.LexiconTypeID = "app.bsky.embed.images#view" + return json.Marshal(t.EmbedImages_View) + } + if t.EmbedVideo_View != nil { + t.EmbedVideo_View.LexiconTypeID = "app.bsky.embed.video#view" + return json.Marshal(t.EmbedVideo_View) + } + if t.EmbedExternal_View != nil { + t.EmbedExternal_View.LexiconTypeID = "app.bsky.embed.external#view" + return json.Marshal(t.EmbedExternal_View) + } + if t.EmbedRecord_View != nil { + t.EmbedRecord_View.LexiconTypeID = "app.bsky.embed.record#view" + return json.Marshal(t.EmbedRecord_View) + } + if t.EmbedRecordWithMedia_View != nil { + t.EmbedRecordWithMedia_View.LexiconTypeID = "app.bsky.embed.recordWithMedia#view" + return json.Marshal(t.EmbedRecordWithMedia_View) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedDefs_PostView_Embed) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.images#view": + t.EmbedImages_View = new(EmbedImages_View) + return json.Unmarshal(b, t.EmbedImages_View) + case "app.bsky.embed.video#view": + t.EmbedVideo_View = new(EmbedVideo_View) + return json.Unmarshal(b, t.EmbedVideo_View) + case "app.bsky.embed.external#view": + t.EmbedExternal_View = new(EmbedExternal_View) + return json.Unmarshal(b, t.EmbedExternal_View) + case "app.bsky.embed.record#view": + t.EmbedRecord_View = new(EmbedRecord_View) + return json.Unmarshal(b, t.EmbedRecord_View) + case "app.bsky.embed.recordWithMedia#view": + t.EmbedRecordWithMedia_View = new(EmbedRecordWithMedia_View) + return json.Unmarshal(b, t.EmbedRecordWithMedia_View) + default: + return nil + } +} + +// FeedDefs_ReasonPin is a "reasonPin" in the app.bsky.feed.defs schema. +type FeedDefs_ReasonPin struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#reasonPin"` +} + +// FeedDefs_ReasonRepost is a "reasonRepost" in the app.bsky.feed.defs schema. +type FeedDefs_ReasonRepost struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#reasonRepost"` + By *ActorDefs_ProfileViewBasic `json:"by" cborgen:"by"` + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Uri *string `json:"uri,omitempty" cborgen:"uri,omitempty"` +} + +// FeedDefs_ReplyRef is a "replyRef" in the app.bsky.feed.defs schema. +type FeedDefs_ReplyRef struct { + // grandparentAuthor: When parent is a reply to another post, this is the author of that post. + GrandparentAuthor *ActorDefs_ProfileViewBasic `json:"grandparentAuthor,omitempty" cborgen:"grandparentAuthor,omitempty"` + Parent *FeedDefs_ReplyRef_Parent `json:"parent" cborgen:"parent"` + Root *FeedDefs_ReplyRef_Root `json:"root" cborgen:"root"` +} + +type FeedDefs_ReplyRef_Parent struct { + FeedDefs_PostView *FeedDefs_PostView + FeedDefs_NotFoundPost *FeedDefs_NotFoundPost + FeedDefs_BlockedPost *FeedDefs_BlockedPost +} + +func (t *FeedDefs_ReplyRef_Parent) MarshalJSON() ([]byte, error) { + if t.FeedDefs_PostView != nil { + t.FeedDefs_PostView.LexiconTypeID = "app.bsky.feed.defs#postView" + return json.Marshal(t.FeedDefs_PostView) + } + if t.FeedDefs_NotFoundPost != nil { + t.FeedDefs_NotFoundPost.LexiconTypeID = "app.bsky.feed.defs#notFoundPost" + return json.Marshal(t.FeedDefs_NotFoundPost) + } + if t.FeedDefs_BlockedPost != nil { + t.FeedDefs_BlockedPost.LexiconTypeID = "app.bsky.feed.defs#blockedPost" + return json.Marshal(t.FeedDefs_BlockedPost) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedDefs_ReplyRef_Parent) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.defs#postView": + t.FeedDefs_PostView = new(FeedDefs_PostView) + return json.Unmarshal(b, t.FeedDefs_PostView) + case "app.bsky.feed.defs#notFoundPost": + t.FeedDefs_NotFoundPost = new(FeedDefs_NotFoundPost) + return json.Unmarshal(b, t.FeedDefs_NotFoundPost) + case "app.bsky.feed.defs#blockedPost": + t.FeedDefs_BlockedPost = new(FeedDefs_BlockedPost) + return json.Unmarshal(b, t.FeedDefs_BlockedPost) + default: + return nil + } +} + +type FeedDefs_ReplyRef_Root struct { + FeedDefs_PostView *FeedDefs_PostView + FeedDefs_NotFoundPost *FeedDefs_NotFoundPost + FeedDefs_BlockedPost *FeedDefs_BlockedPost +} + +func (t *FeedDefs_ReplyRef_Root) MarshalJSON() ([]byte, error) { + if t.FeedDefs_PostView != nil { + t.FeedDefs_PostView.LexiconTypeID = "app.bsky.feed.defs#postView" + return json.Marshal(t.FeedDefs_PostView) + } + if t.FeedDefs_NotFoundPost != nil { + t.FeedDefs_NotFoundPost.LexiconTypeID = "app.bsky.feed.defs#notFoundPost" + return json.Marshal(t.FeedDefs_NotFoundPost) + } + if t.FeedDefs_BlockedPost != nil { + t.FeedDefs_BlockedPost.LexiconTypeID = "app.bsky.feed.defs#blockedPost" + return json.Marshal(t.FeedDefs_BlockedPost) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedDefs_ReplyRef_Root) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.defs#postView": + t.FeedDefs_PostView = new(FeedDefs_PostView) + return json.Unmarshal(b, t.FeedDefs_PostView) + case "app.bsky.feed.defs#notFoundPost": + t.FeedDefs_NotFoundPost = new(FeedDefs_NotFoundPost) + return json.Unmarshal(b, t.FeedDefs_NotFoundPost) + case "app.bsky.feed.defs#blockedPost": + t.FeedDefs_BlockedPost = new(FeedDefs_BlockedPost) + return json.Unmarshal(b, t.FeedDefs_BlockedPost) + default: + return nil + } +} + +// FeedDefs_SkeletonFeedPost is a "skeletonFeedPost" in the app.bsky.feed.defs schema. +type FeedDefs_SkeletonFeedPost struct { + // feedContext: Context that will be passed through to client and may be passed to feed generator back alongside interactions. + FeedContext *string `json:"feedContext,omitempty" cborgen:"feedContext,omitempty"` + Post string `json:"post" cborgen:"post"` + Reason *FeedDefs_SkeletonFeedPost_Reason `json:"reason,omitempty" cborgen:"reason,omitempty"` +} + +type FeedDefs_SkeletonFeedPost_Reason struct { + FeedDefs_SkeletonReasonRepost *FeedDefs_SkeletonReasonRepost + FeedDefs_SkeletonReasonPin *FeedDefs_SkeletonReasonPin +} + +func (t *FeedDefs_SkeletonFeedPost_Reason) MarshalJSON() ([]byte, error) { + if t.FeedDefs_SkeletonReasonRepost != nil { + t.FeedDefs_SkeletonReasonRepost.LexiconTypeID = "app.bsky.feed.defs#skeletonReasonRepost" + return json.Marshal(t.FeedDefs_SkeletonReasonRepost) + } + if t.FeedDefs_SkeletonReasonPin != nil { + t.FeedDefs_SkeletonReasonPin.LexiconTypeID = "app.bsky.feed.defs#skeletonReasonPin" + return json.Marshal(t.FeedDefs_SkeletonReasonPin) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedDefs_SkeletonFeedPost_Reason) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.defs#skeletonReasonRepost": + t.FeedDefs_SkeletonReasonRepost = new(FeedDefs_SkeletonReasonRepost) + return json.Unmarshal(b, t.FeedDefs_SkeletonReasonRepost) + case "app.bsky.feed.defs#skeletonReasonPin": + t.FeedDefs_SkeletonReasonPin = new(FeedDefs_SkeletonReasonPin) + return json.Unmarshal(b, t.FeedDefs_SkeletonReasonPin) + default: + return nil + } +} + +// FeedDefs_SkeletonReasonPin is a "skeletonReasonPin" in the app.bsky.feed.defs schema. +type FeedDefs_SkeletonReasonPin struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#skeletonReasonPin"` +} + +// FeedDefs_SkeletonReasonRepost is a "skeletonReasonRepost" in the app.bsky.feed.defs schema. +type FeedDefs_SkeletonReasonRepost struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#skeletonReasonRepost"` + Repost string `json:"repost" cborgen:"repost"` +} + +// FeedDefs_ThreadContext is a "threadContext" in the app.bsky.feed.defs schema. +// +// Metadata about this post within the context of the thread it is in. +type FeedDefs_ThreadContext struct { + RootAuthorLike *string `json:"rootAuthorLike,omitempty" cborgen:"rootAuthorLike,omitempty"` +} + +// FeedDefs_ThreadViewPost is a "threadViewPost" in the app.bsky.feed.defs schema. +type FeedDefs_ThreadViewPost struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.defs#threadViewPost"` + Parent *FeedDefs_ThreadViewPost_Parent `json:"parent,omitempty" cborgen:"parent,omitempty"` + Post *FeedDefs_PostView `json:"post" cborgen:"post"` + Replies []*FeedDefs_ThreadViewPost_Replies_Elem `json:"replies,omitempty" cborgen:"replies,omitempty"` + ThreadContext *FeedDefs_ThreadContext `json:"threadContext,omitempty" cborgen:"threadContext,omitempty"` +} + +type FeedDefs_ThreadViewPost_Parent struct { + FeedDefs_ThreadViewPost *FeedDefs_ThreadViewPost + FeedDefs_NotFoundPost *FeedDefs_NotFoundPost + FeedDefs_BlockedPost *FeedDefs_BlockedPost +} + +func (t *FeedDefs_ThreadViewPost_Parent) MarshalJSON() ([]byte, error) { + if t.FeedDefs_ThreadViewPost != nil { + t.FeedDefs_ThreadViewPost.LexiconTypeID = "app.bsky.feed.defs#threadViewPost" + return json.Marshal(t.FeedDefs_ThreadViewPost) + } + if t.FeedDefs_NotFoundPost != nil { + t.FeedDefs_NotFoundPost.LexiconTypeID = "app.bsky.feed.defs#notFoundPost" + return json.Marshal(t.FeedDefs_NotFoundPost) + } + if t.FeedDefs_BlockedPost != nil { + t.FeedDefs_BlockedPost.LexiconTypeID = "app.bsky.feed.defs#blockedPost" + return json.Marshal(t.FeedDefs_BlockedPost) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedDefs_ThreadViewPost_Parent) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.defs#threadViewPost": + t.FeedDefs_ThreadViewPost = new(FeedDefs_ThreadViewPost) + return json.Unmarshal(b, t.FeedDefs_ThreadViewPost) + case "app.bsky.feed.defs#notFoundPost": + t.FeedDefs_NotFoundPost = new(FeedDefs_NotFoundPost) + return json.Unmarshal(b, t.FeedDefs_NotFoundPost) + case "app.bsky.feed.defs#blockedPost": + t.FeedDefs_BlockedPost = new(FeedDefs_BlockedPost) + return json.Unmarshal(b, t.FeedDefs_BlockedPost) + default: + return nil + } +} + +type FeedDefs_ThreadViewPost_Replies_Elem struct { + FeedDefs_ThreadViewPost *FeedDefs_ThreadViewPost + FeedDefs_NotFoundPost *FeedDefs_NotFoundPost + FeedDefs_BlockedPost *FeedDefs_BlockedPost +} + +func (t *FeedDefs_ThreadViewPost_Replies_Elem) MarshalJSON() ([]byte, error) { + if t.FeedDefs_ThreadViewPost != nil { + t.FeedDefs_ThreadViewPost.LexiconTypeID = "app.bsky.feed.defs#threadViewPost" + return json.Marshal(t.FeedDefs_ThreadViewPost) + } + if t.FeedDefs_NotFoundPost != nil { + t.FeedDefs_NotFoundPost.LexiconTypeID = "app.bsky.feed.defs#notFoundPost" + return json.Marshal(t.FeedDefs_NotFoundPost) + } + if t.FeedDefs_BlockedPost != nil { + t.FeedDefs_BlockedPost.LexiconTypeID = "app.bsky.feed.defs#blockedPost" + return json.Marshal(t.FeedDefs_BlockedPost) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedDefs_ThreadViewPost_Replies_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.defs#threadViewPost": + t.FeedDefs_ThreadViewPost = new(FeedDefs_ThreadViewPost) + return json.Unmarshal(b, t.FeedDefs_ThreadViewPost) + case "app.bsky.feed.defs#notFoundPost": + t.FeedDefs_NotFoundPost = new(FeedDefs_NotFoundPost) + return json.Unmarshal(b, t.FeedDefs_NotFoundPost) + case "app.bsky.feed.defs#blockedPost": + t.FeedDefs_BlockedPost = new(FeedDefs_BlockedPost) + return json.Unmarshal(b, t.FeedDefs_BlockedPost) + default: + return nil + } +} + +// FeedDefs_ThreadgateView is a "threadgateView" in the app.bsky.feed.defs schema. +type FeedDefs_ThreadgateView struct { + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Lists []*GraphDefs_ListViewBasic `json:"lists,omitempty" cborgen:"lists,omitempty"` + Record *lexutil.LexiconTypeDecoder `json:"record,omitempty" cborgen:"record,omitempty"` + Uri *string `json:"uri,omitempty" cborgen:"uri,omitempty"` +} + +// FeedDefs_ViewerState is a "viewerState" in the app.bsky.feed.defs schema. +// +// Metadata about the requesting account's relationship with the subject content. Only has meaningful content for authed requests. +type FeedDefs_ViewerState struct { + Bookmarked *bool `json:"bookmarked,omitempty" cborgen:"bookmarked,omitempty"` + EmbeddingDisabled *bool `json:"embeddingDisabled,omitempty" cborgen:"embeddingDisabled,omitempty"` + Like *string `json:"like,omitempty" cborgen:"like,omitempty"` + Pinned *bool `json:"pinned,omitempty" cborgen:"pinned,omitempty"` + ReplyDisabled *bool `json:"replyDisabled,omitempty" cborgen:"replyDisabled,omitempty"` + Repost *string `json:"repost,omitempty" cborgen:"repost,omitempty"` + ThreadMuted *bool `json:"threadMuted,omitempty" cborgen:"threadMuted,omitempty"` +} diff --git a/api/bsky/feeddescribeFeedGenerator.go b/api/bsky/feeddescribeFeedGenerator.go new file mode 100644 index 000000000..c638781e9 --- /dev/null +++ b/api/bsky/feeddescribeFeedGenerator.go @@ -0,0 +1,39 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.describeFeedGenerator + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedDescribeFeedGenerator_Feed is a "feed" in the app.bsky.feed.describeFeedGenerator schema. +type FeedDescribeFeedGenerator_Feed struct { + Uri string `json:"uri" cborgen:"uri"` +} + +// FeedDescribeFeedGenerator_Links is a "links" in the app.bsky.feed.describeFeedGenerator schema. +type FeedDescribeFeedGenerator_Links struct { + PrivacyPolicy *string `json:"privacyPolicy,omitempty" cborgen:"privacyPolicy,omitempty"` + TermsOfService *string `json:"termsOfService,omitempty" cborgen:"termsOfService,omitempty"` +} + +// FeedDescribeFeedGenerator_Output is the output of a app.bsky.feed.describeFeedGenerator call. +type FeedDescribeFeedGenerator_Output struct { + Did string `json:"did" cborgen:"did"` + Feeds []*FeedDescribeFeedGenerator_Feed `json:"feeds" cborgen:"feeds"` + Links *FeedDescribeFeedGenerator_Links `json:"links,omitempty" cborgen:"links,omitempty"` +} + +// FeedDescribeFeedGenerator calls the XRPC method "app.bsky.feed.describeFeedGenerator". +func FeedDescribeFeedGenerator(ctx context.Context, c lexutil.LexClient) (*FeedDescribeFeedGenerator_Output, error) { + var out FeedDescribeFeedGenerator_Output + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.describeFeedGenerator", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedembed.go b/api/bsky/feedembed.go deleted file mode 100644 index 3029a3b17..000000000 --- a/api/bsky/feedembed.go +++ /dev/null @@ -1,77 +0,0 @@ -package schemagen - -import ( - "encoding/json" - "fmt" - - "github.com/bluesky-social/indigo/lex/util" -) - -// schema: app.bsky.feed.embed - -func init() { -} - -type FeedEmbed_Record struct { - Record any `json:"record" cborgen:"record"` - Type string `json:"type" cborgen:"type"` - Author *ActorRef_WithInfo `json:"author" cborgen:"author"` -} - -type FeedEmbed_External struct { - ImageUri string `json:"imageUri" cborgen:"imageUri"` - Type string `json:"type" cborgen:"type"` - Uri string `json:"uri" cborgen:"uri"` - Title string `json:"title" cborgen:"title"` - Description string `json:"description" cborgen:"description"` -} - -type FeedEmbed struct { - Items []*FeedEmbed_Items_Elem `json:"items" cborgen:"items"` -} - -type FeedEmbed_Items_Elem struct { - FeedEmbed_Media *FeedEmbed_Media - FeedEmbed_Record *FeedEmbed_Record - FeedEmbed_External *FeedEmbed_External -} - -func (t *FeedEmbed_Items_Elem) MarshalJSON() ([]byte, error) { - if t.FeedEmbed_Media != nil { - return json.Marshal(t.FeedEmbed_Media) - } - if t.FeedEmbed_Record != nil { - return json.Marshal(t.FeedEmbed_Record) - } - if t.FeedEmbed_External != nil { - return json.Marshal(t.FeedEmbed_External) - } - return nil, fmt.Errorf("cannot marshal empty enum") -} -func (t *FeedEmbed_Items_Elem) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) - if err != nil { - return err - } - - switch typ { - case "app.bsky.feed.embed#media": - t.FeedEmbed_Media = new(FeedEmbed_Media) - return json.Unmarshal(b, t.FeedEmbed_Media) - case "app.bsky.feed.embed#record": - t.FeedEmbed_Record = new(FeedEmbed_Record) - return json.Unmarshal(b, t.FeedEmbed_Record) - case "app.bsky.feed.embed#external": - t.FeedEmbed_External = new(FeedEmbed_External) - return json.Unmarshal(b, t.FeedEmbed_External) - - default: - return nil - } -} - -type FeedEmbed_Media struct { - Alt *string `json:"alt" cborgen:"alt"` - Thumb *util.Blob `json:"thumb" cborgen:"thumb"` - Original *util.Blob `json:"original" cborgen:"original"` -} diff --git a/api/bsky/feedfeedViewPost.go b/api/bsky/feedfeedViewPost.go deleted file mode 100644 index 10b19ecad..000000000 --- a/api/bsky/feedfeedViewPost.go +++ /dev/null @@ -1,73 +0,0 @@ -package schemagen - -import ( - "encoding/json" - "fmt" - - "github.com/bluesky-social/indigo/lex/util" -) - -// schema: app.bsky.feed.feedViewPost - -func init() { -} - -type FeedFeedViewPost struct { - LexiconTypeID string `json:"$type,omitempty"` - Post *FeedPost_View `json:"post" cborgen:"post"` - Reason *FeedFeedViewPost_Reason `json:"reason,omitempty" cborgen:"reason"` - Reply *FeedFeedViewPost_ReplyRef `json:"reply,omitempty" cborgen:"reply"` -} - -type FeedFeedViewPost_Reason struct { - FeedFeedViewPost_ReasonTrend *FeedFeedViewPost_ReasonTrend - FeedFeedViewPost_ReasonRepost *FeedFeedViewPost_ReasonRepost -} - -func (t *FeedFeedViewPost_Reason) MarshalJSON() ([]byte, error) { - if t.FeedFeedViewPost_ReasonTrend != nil { - t.FeedFeedViewPost_ReasonTrend.LexiconTypeID = "app.bsky.feed.feedViewPost#reasonTrend" - return json.Marshal(t.FeedFeedViewPost_ReasonTrend) - } - if t.FeedFeedViewPost_ReasonRepost != nil { - t.FeedFeedViewPost_ReasonRepost.LexiconTypeID = "app.bsky.feed.feedViewPost#reasonRepost" - return json.Marshal(t.FeedFeedViewPost_ReasonRepost) - } - return nil, fmt.Errorf("cannot marshal empty enum") -} -func (t *FeedFeedViewPost_Reason) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) - if err != nil { - return err - } - - switch typ { - case "app.bsky.feed.feedViewPost#reasonTrend": - t.FeedFeedViewPost_ReasonTrend = new(FeedFeedViewPost_ReasonTrend) - return json.Unmarshal(b, t.FeedFeedViewPost_ReasonTrend) - case "app.bsky.feed.feedViewPost#reasonRepost": - t.FeedFeedViewPost_ReasonRepost = new(FeedFeedViewPost_ReasonRepost) - return json.Unmarshal(b, t.FeedFeedViewPost_ReasonRepost) - - default: - return nil - } -} - -type FeedFeedViewPost_ReasonRepost struct { - LexiconTypeID string `json:"$type,omitempty"` - By *ActorRef_WithInfo `json:"by" cborgen:"by"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` -} - -type FeedFeedViewPost_ReasonTrend struct { - LexiconTypeID string `json:"$type,omitempty"` - By *ActorRef_WithInfo `json:"by" cborgen:"by"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` -} - -type FeedFeedViewPost_ReplyRef struct { - LexiconTypeID string `json:"$type,omitempty"` - Parent *FeedPost_View `json:"parent" cborgen:"parent"` - Root *FeedPost_View `json:"root" cborgen:"root"` -} diff --git a/api/bsky/feedgenerator.go b/api/bsky/feedgenerator.go new file mode 100644 index 000000000..e8dc150d7 --- /dev/null +++ b/api/bsky/feedgenerator.go @@ -0,0 +1,90 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.generator + +package bsky + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func init() { + lexutil.RegisterType("app.bsky.feed.generator", &FeedGenerator{}) +} + +type FeedGenerator struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.generator"` + // acceptsInteractions: Declaration that a feed accepts feedback interactions from a client through app.bsky.feed.sendInteractions + AcceptsInteractions *bool `json:"acceptsInteractions,omitempty" cborgen:"acceptsInteractions,omitempty"` + Avatar *lexutil.LexBlob `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + ContentMode *string `json:"contentMode,omitempty" cborgen:"contentMode,omitempty"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + DescriptionFacets []*RichtextFacet `json:"descriptionFacets,omitempty" cborgen:"descriptionFacets,omitempty"` + Did string `json:"did" cborgen:"did"` + DisplayName string `json:"displayName" cborgen:"displayName"` + // labels: Self-label values + Labels *FeedGenerator_Labels `json:"labels,omitempty" cborgen:"labels,omitempty"` +} + +// Self-label values +type FeedGenerator_Labels struct { + LabelDefs_SelfLabels *comatproto.LabelDefs_SelfLabels +} + +func (t *FeedGenerator_Labels) MarshalJSON() ([]byte, error) { + if t.LabelDefs_SelfLabels != nil { + t.LabelDefs_SelfLabels.LexiconTypeID = "com.atproto.label.defs#selfLabels" + return json.Marshal(t.LabelDefs_SelfLabels) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedGenerator_Labels) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return json.Unmarshal(b, t.LabelDefs_SelfLabels) + default: + return nil + } +} + +func (t *FeedGenerator_Labels) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.LabelDefs_SelfLabels != nil { + return t.LabelDefs_SelfLabels.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *FeedGenerator_Labels) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return t.LabelDefs_SelfLabels.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } +} diff --git a/api/bsky/feedgetActorFeeds.go b/api/bsky/feedgetActorFeeds.go new file mode 100644 index 000000000..f5bf7786d --- /dev/null +++ b/api/bsky/feedgetActorFeeds.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getActorFeeds + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetActorFeeds_Output is the output of a app.bsky.feed.getActorFeeds call. +type FeedGetActorFeeds_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feeds []*FeedDefs_GeneratorView `json:"feeds" cborgen:"feeds"` +} + +// FeedGetActorFeeds calls the XRPC method "app.bsky.feed.getActorFeeds". +func FeedGetActorFeeds(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64) (*FeedGetActorFeeds_Output, error) { + var out FeedGetActorFeeds_Output + + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getActorFeeds", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetActorLikes.go b/api/bsky/feedgetActorLikes.go new file mode 100644 index 000000000..8ae0fc092 --- /dev/null +++ b/api/bsky/feedgetActorLikes.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getActorLikes + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetActorLikes_Output is the output of a app.bsky.feed.getActorLikes call. +type FeedGetActorLikes_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feed []*FeedDefs_FeedViewPost `json:"feed" cborgen:"feed"` +} + +// FeedGetActorLikes calls the XRPC method "app.bsky.feed.getActorLikes". +func FeedGetActorLikes(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64) (*FeedGetActorLikes_Output, error) { + var out FeedGetActorLikes_Output + + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getActorLikes", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetAuthorFeed.go b/api/bsky/feedgetAuthorFeed.go index 31461e080..f22b48a93 100644 --- a/api/bsky/feedgetAuthorFeed.go +++ b/api/bsky/feedgetAuthorFeed.go @@ -1,31 +1,42 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getAuthorFeed + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.feed.getAuthorFeed - -func init() { -} - +// FeedGetAuthorFeed_Output is the output of a app.bsky.feed.getAuthorFeed call. type FeedGetAuthorFeed_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Feed []*FeedFeedViewPost `json:"feed" cborgen:"feed"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feed []*FeedDefs_FeedViewPost `json:"feed" cborgen:"feed"` } -func FeedGetAuthorFeed(ctx context.Context, c *xrpc.Client, author string, before string, limit int64) (*FeedGetAuthorFeed_Output, error) { +// FeedGetAuthorFeed calls the XRPC method "app.bsky.feed.getAuthorFeed". +// +// filter: Combinations of post/repost types to include in response. +func FeedGetAuthorFeed(ctx context.Context, c lexutil.LexClient, actor string, cursor string, filter string, includePins bool, limit int64) (*FeedGetAuthorFeed_Output, error) { var out FeedGetAuthorFeed_Output - params := map[string]interface{}{ - "author": author, - "before": before, - "limit": limit, + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if filter != "" { + params["filter"] = filter + } + if includePins { + params["includePins"] = includePins + } + if limit != 0 { + params["limit"] = limit } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getAuthorFeed", params, nil, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getAuthorFeed", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/feedgetFeed.go b/api/bsky/feedgetFeed.go new file mode 100644 index 000000000..a7f30906e --- /dev/null +++ b/api/bsky/feedgetFeed.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getFeed + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetFeed_Output is the output of a app.bsky.feed.getFeed call. +type FeedGetFeed_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feed []*FeedDefs_FeedViewPost `json:"feed" cborgen:"feed"` +} + +// FeedGetFeed calls the XRPC method "app.bsky.feed.getFeed". +func FeedGetFeed(ctx context.Context, c lexutil.LexClient, cursor string, feed string, limit int64) (*FeedGetFeed_Output, error) { + var out FeedGetFeed_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + params["feed"] = feed + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getFeed", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetFeedGenerator.go b/api/bsky/feedgetFeedGenerator.go new file mode 100644 index 000000000..d11ab55b1 --- /dev/null +++ b/api/bsky/feedgetFeedGenerator.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getFeedGenerator + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetFeedGenerator_Output is the output of a app.bsky.feed.getFeedGenerator call. +type FeedGetFeedGenerator_Output struct { + // isOnline: Indicates whether the feed generator service has been online recently, or else seems to be inactive. + IsOnline bool `json:"isOnline" cborgen:"isOnline"` + // isValid: Indicates whether the feed generator service is compatible with the record declaration. + IsValid bool `json:"isValid" cborgen:"isValid"` + View *FeedDefs_GeneratorView `json:"view" cborgen:"view"` +} + +// FeedGetFeedGenerator calls the XRPC method "app.bsky.feed.getFeedGenerator". +// +// feed: AT-URI of the feed generator record. +func FeedGetFeedGenerator(ctx context.Context, c lexutil.LexClient, feed string) (*FeedGetFeedGenerator_Output, error) { + var out FeedGetFeedGenerator_Output + + params := map[string]interface{}{} + params["feed"] = feed + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getFeedGenerator", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetFeedGenerators.go b/api/bsky/feedgetFeedGenerators.go new file mode 100644 index 000000000..b1d05365c --- /dev/null +++ b/api/bsky/feedgetFeedGenerators.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getFeedGenerators + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetFeedGenerators_Output is the output of a app.bsky.feed.getFeedGenerators call. +type FeedGetFeedGenerators_Output struct { + Feeds []*FeedDefs_GeneratorView `json:"feeds" cborgen:"feeds"` +} + +// FeedGetFeedGenerators calls the XRPC method "app.bsky.feed.getFeedGenerators". +func FeedGetFeedGenerators(ctx context.Context, c lexutil.LexClient, feeds []string) (*FeedGetFeedGenerators_Output, error) { + var out FeedGetFeedGenerators_Output + + params := map[string]interface{}{} + params["feeds"] = feeds + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getFeedGenerators", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetFeedSkeleton.go b/api/bsky/feedgetFeedSkeleton.go new file mode 100644 index 000000000..d8f2ecb7f --- /dev/null +++ b/api/bsky/feedgetFeedSkeleton.go @@ -0,0 +1,40 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getFeedSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetFeedSkeleton_Output is the output of a app.bsky.feed.getFeedSkeleton call. +type FeedGetFeedSkeleton_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feed []*FeedDefs_SkeletonFeedPost `json:"feed" cborgen:"feed"` + // reqId: Unique identifier per request that may be passed back alongside interactions. + ReqId *string `json:"reqId,omitempty" cborgen:"reqId,omitempty"` +} + +// FeedGetFeedSkeleton calls the XRPC method "app.bsky.feed.getFeedSkeleton". +// +// feed: Reference to feed generator record describing the specific feed being requested. +func FeedGetFeedSkeleton(ctx context.Context, c lexutil.LexClient, cursor string, feed string, limit int64) (*FeedGetFeedSkeleton_Output, error) { + var out FeedGetFeedSkeleton_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + params["feed"] = feed + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getFeedSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetLikes.go b/api/bsky/feedgetLikes.go new file mode 100644 index 000000000..1dcab7347 --- /dev/null +++ b/api/bsky/feedgetLikes.go @@ -0,0 +1,51 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getLikes + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetLikes_Like is a "like" in the app.bsky.feed.getLikes schema. +type FeedGetLikes_Like struct { + Actor *ActorDefs_ProfileView `json:"actor" cborgen:"actor"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` +} + +// FeedGetLikes_Output is the output of a app.bsky.feed.getLikes call. +type FeedGetLikes_Output struct { + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Likes []*FeedGetLikes_Like `json:"likes" cborgen:"likes"` + Uri string `json:"uri" cborgen:"uri"` +} + +// FeedGetLikes calls the XRPC method "app.bsky.feed.getLikes". +// +// cid: CID of the subject record (aka, specific version of record), to filter likes. +// uri: AT-URI of the subject (eg, a post record). +func FeedGetLikes(ctx context.Context, c lexutil.LexClient, cid string, cursor string, limit int64, uri string) (*FeedGetLikes_Output, error) { + var out FeedGetLikes_Output + + params := map[string]interface{}{} + if cid != "" { + params["cid"] = cid + } + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["uri"] = uri + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getLikes", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetListFeed.go b/api/bsky/feedgetListFeed.go new file mode 100644 index 000000000..bd1273a53 --- /dev/null +++ b/api/bsky/feedgetListFeed.go @@ -0,0 +1,38 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getListFeed + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetListFeed_Output is the output of a app.bsky.feed.getListFeed call. +type FeedGetListFeed_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feed []*FeedDefs_FeedViewPost `json:"feed" cborgen:"feed"` +} + +// FeedGetListFeed calls the XRPC method "app.bsky.feed.getListFeed". +// +// list: Reference (AT-URI) to the list record. +func FeedGetListFeed(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, list string) (*FeedGetListFeed_Output, error) { + var out FeedGetListFeed_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["list"] = list + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getListFeed", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetPostThread.go b/api/bsky/feedgetPostThread.go index 6bb649b4a..f702fe412 100644 --- a/api/bsky/feedgetPostThread.go +++ b/api/bsky/feedgetPostThread.go @@ -1,150 +1,83 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getPostThread + +package bsky import ( "context" "encoding/json" "fmt" - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.feed.getPostThread - -func init() { -} - -type FeedGetPostThread_NotFoundPost struct { - LexiconTypeID string `json:"$type,omitempty"` - NotFound bool `json:"notFound" cborgen:"notFound"` - Uri string `json:"uri" cborgen:"uri"` -} - +// FeedGetPostThread_Output is the output of a app.bsky.feed.getPostThread call. type FeedGetPostThread_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Thread *FeedGetPostThread_Output_Thread `json:"thread" cborgen:"thread"` + Thread *FeedGetPostThread_Output_Thread `json:"thread" cborgen:"thread"` + Threadgate *FeedDefs_ThreadgateView `json:"threadgate,omitempty" cborgen:"threadgate,omitempty"` } type FeedGetPostThread_Output_Thread struct { - FeedGetPostThread_ThreadViewPost *FeedGetPostThread_ThreadViewPost - FeedGetPostThread_NotFoundPost *FeedGetPostThread_NotFoundPost + FeedDefs_ThreadViewPost *FeedDefs_ThreadViewPost + FeedDefs_NotFoundPost *FeedDefs_NotFoundPost + FeedDefs_BlockedPost *FeedDefs_BlockedPost } func (t *FeedGetPostThread_Output_Thread) MarshalJSON() ([]byte, error) { - if t.FeedGetPostThread_ThreadViewPost != nil { - t.FeedGetPostThread_ThreadViewPost.LexiconTypeID = "app.bsky.feed.getPostThread#threadViewPost" - return json.Marshal(t.FeedGetPostThread_ThreadViewPost) - } - if t.FeedGetPostThread_NotFoundPost != nil { - t.FeedGetPostThread_NotFoundPost.LexiconTypeID = "app.bsky.feed.getPostThread#notFoundPost" - return json.Marshal(t.FeedGetPostThread_NotFoundPost) + if t.FeedDefs_ThreadViewPost != nil { + t.FeedDefs_ThreadViewPost.LexiconTypeID = "app.bsky.feed.defs#threadViewPost" + return json.Marshal(t.FeedDefs_ThreadViewPost) } - return nil, fmt.Errorf("cannot marshal empty enum") -} -func (t *FeedGetPostThread_Output_Thread) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) - if err != nil { - return err + if t.FeedDefs_NotFoundPost != nil { + t.FeedDefs_NotFoundPost.LexiconTypeID = "app.bsky.feed.defs#notFoundPost" + return json.Marshal(t.FeedDefs_NotFoundPost) } - - switch typ { - case "app.bsky.feed.getPostThread#threadViewPost": - t.FeedGetPostThread_ThreadViewPost = new(FeedGetPostThread_ThreadViewPost) - return json.Unmarshal(b, t.FeedGetPostThread_ThreadViewPost) - case "app.bsky.feed.getPostThread#notFoundPost": - t.FeedGetPostThread_NotFoundPost = new(FeedGetPostThread_NotFoundPost) - return json.Unmarshal(b, t.FeedGetPostThread_NotFoundPost) - - default: - return nil + if t.FeedDefs_BlockedPost != nil { + t.FeedDefs_BlockedPost.LexiconTypeID = "app.bsky.feed.defs#blockedPost" + return json.Marshal(t.FeedDefs_BlockedPost) } + return nil, fmt.Errorf("can not marshal empty union as JSON") } -type FeedGetPostThread_ThreadViewPost struct { - LexiconTypeID string `json:"$type,omitempty"` - Parent *FeedGetPostThread_ThreadViewPost_Parent `json:"parent,omitempty" cborgen:"parent"` - Post *FeedPost_View `json:"post" cborgen:"post"` - Replies []*FeedGetPostThread_ThreadViewPost_Replies_Elem `json:"replies,omitempty" cborgen:"replies"` -} - -type FeedGetPostThread_ThreadViewPost_Parent struct { - FeedGetPostThread_ThreadViewPost *FeedGetPostThread_ThreadViewPost - FeedGetPostThread_NotFoundPost *FeedGetPostThread_NotFoundPost -} - -func (t *FeedGetPostThread_ThreadViewPost_Parent) MarshalJSON() ([]byte, error) { - if t.FeedGetPostThread_ThreadViewPost != nil { - t.FeedGetPostThread_ThreadViewPost.LexiconTypeID = "app.bsky.feed.getPostThread#threadViewPost" - return json.Marshal(t.FeedGetPostThread_ThreadViewPost) - } - if t.FeedGetPostThread_NotFoundPost != nil { - t.FeedGetPostThread_NotFoundPost.LexiconTypeID = "app.bsky.feed.getPostThread#notFoundPost" - return json.Marshal(t.FeedGetPostThread_NotFoundPost) - } - return nil, fmt.Errorf("cannot marshal empty enum") -} -func (t *FeedGetPostThread_ThreadViewPost_Parent) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) - if err != nil { - return err - } - - switch typ { - case "app.bsky.feed.getPostThread#threadViewPost": - t.FeedGetPostThread_ThreadViewPost = new(FeedGetPostThread_ThreadViewPost) - return json.Unmarshal(b, t.FeedGetPostThread_ThreadViewPost) - case "app.bsky.feed.getPostThread#notFoundPost": - t.FeedGetPostThread_NotFoundPost = new(FeedGetPostThread_NotFoundPost) - return json.Unmarshal(b, t.FeedGetPostThread_NotFoundPost) - - default: - return nil - } -} - -type FeedGetPostThread_ThreadViewPost_Replies_Elem struct { - FeedGetPostThread_ThreadViewPost *FeedGetPostThread_ThreadViewPost - FeedGetPostThread_NotFoundPost *FeedGetPostThread_NotFoundPost -} - -func (t *FeedGetPostThread_ThreadViewPost_Replies_Elem) MarshalJSON() ([]byte, error) { - if t.FeedGetPostThread_ThreadViewPost != nil { - t.FeedGetPostThread_ThreadViewPost.LexiconTypeID = "app.bsky.feed.getPostThread#threadViewPost" - return json.Marshal(t.FeedGetPostThread_ThreadViewPost) - } - if t.FeedGetPostThread_NotFoundPost != nil { - t.FeedGetPostThread_NotFoundPost.LexiconTypeID = "app.bsky.feed.getPostThread#notFoundPost" - return json.Marshal(t.FeedGetPostThread_NotFoundPost) - } - return nil, fmt.Errorf("cannot marshal empty enum") -} -func (t *FeedGetPostThread_ThreadViewPost_Replies_Elem) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) +func (t *FeedGetPostThread_Output_Thread) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) if err != nil { return err } switch typ { - case "app.bsky.feed.getPostThread#threadViewPost": - t.FeedGetPostThread_ThreadViewPost = new(FeedGetPostThread_ThreadViewPost) - return json.Unmarshal(b, t.FeedGetPostThread_ThreadViewPost) - case "app.bsky.feed.getPostThread#notFoundPost": - t.FeedGetPostThread_NotFoundPost = new(FeedGetPostThread_NotFoundPost) - return json.Unmarshal(b, t.FeedGetPostThread_NotFoundPost) - + case "app.bsky.feed.defs#threadViewPost": + t.FeedDefs_ThreadViewPost = new(FeedDefs_ThreadViewPost) + return json.Unmarshal(b, t.FeedDefs_ThreadViewPost) + case "app.bsky.feed.defs#notFoundPost": + t.FeedDefs_NotFoundPost = new(FeedDefs_NotFoundPost) + return json.Unmarshal(b, t.FeedDefs_NotFoundPost) + case "app.bsky.feed.defs#blockedPost": + t.FeedDefs_BlockedPost = new(FeedDefs_BlockedPost) + return json.Unmarshal(b, t.FeedDefs_BlockedPost) default: return nil } } -func FeedGetPostThread(ctx context.Context, c *xrpc.Client, depth int64, uri string) (*FeedGetPostThread_Output, error) { +// FeedGetPostThread calls the XRPC method "app.bsky.feed.getPostThread". +// +// depth: How many levels of reply depth should be included in response. +// parentHeight: How many levels of parent (and grandparent, etc) post to include. +// uri: Reference (AT-URI) to post record. +func FeedGetPostThread(ctx context.Context, c lexutil.LexClient, depth int64, parentHeight int64, uri string) (*FeedGetPostThread_Output, error) { var out FeedGetPostThread_Output - params := map[string]interface{}{ - "depth": depth, - "uri": uri, + params := map[string]interface{}{} + if depth != 0 { + params["depth"] = depth + } + if parentHeight != 0 { + params["parentHeight"] = parentHeight } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getPostThread", params, nil, &out); err != nil { + params["uri"] = uri + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getPostThread", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/feedgetPosts.go b/api/bsky/feedgetPosts.go new file mode 100644 index 000000000..8d5a3e5ec --- /dev/null +++ b/api/bsky/feedgetPosts.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getPosts + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetPosts_Output is the output of a app.bsky.feed.getPosts call. +type FeedGetPosts_Output struct { + Posts []*FeedDefs_PostView `json:"posts" cborgen:"posts"` +} + +// FeedGetPosts calls the XRPC method "app.bsky.feed.getPosts". +// +// uris: List of post AT-URIs to return hydrated views for. +func FeedGetPosts(ctx context.Context, c lexutil.LexClient, uris []string) (*FeedGetPosts_Output, error) { + var out FeedGetPosts_Output + + params := map[string]interface{}{} + params["uris"] = uris + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getPosts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetQuotes.go b/api/bsky/feedgetQuotes.go new file mode 100644 index 000000000..002b4127f --- /dev/null +++ b/api/bsky/feedgetQuotes.go @@ -0,0 +1,44 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getQuotes + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetQuotes_Output is the output of a app.bsky.feed.getQuotes call. +type FeedGetQuotes_Output struct { + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Posts []*FeedDefs_PostView `json:"posts" cborgen:"posts"` + Uri string `json:"uri" cborgen:"uri"` +} + +// FeedGetQuotes calls the XRPC method "app.bsky.feed.getQuotes". +// +// cid: If supplied, filters to quotes of specific version (by CID) of the post record. +// uri: Reference (AT-URI) of post record +func FeedGetQuotes(ctx context.Context, c lexutil.LexClient, cid string, cursor string, limit int64, uri string) (*FeedGetQuotes_Output, error) { + var out FeedGetQuotes_Output + + params := map[string]interface{}{} + if cid != "" { + params["cid"] = cid + } + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["uri"] = uri + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getQuotes", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetRepostedBy.go b/api/bsky/feedgetRepostedBy.go index ea9f6728c..7559fca21 100644 --- a/api/bsky/feedgetRepostedBy.go +++ b/api/bsky/feedgetRepostedBy.go @@ -1,45 +1,42 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getRepostedBy + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.feed.getRepostedBy - -func init() { -} - +// FeedGetRepostedBy_Output is the output of a app.bsky.feed.getRepostedBy call. type FeedGetRepostedBy_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid *string `json:"cid,omitempty" cborgen:"cid"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - RepostedBy []*FeedGetRepostedBy_RepostedBy `json:"repostedBy" cborgen:"repostedBy"` - Uri string `json:"uri" cborgen:"uri"` + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + RepostedBy []*ActorDefs_ProfileView `json:"repostedBy" cborgen:"repostedBy"` + Uri string `json:"uri" cborgen:"uri"` } -type FeedGetRepostedBy_RepostedBy struct { - LexiconTypeID string `json:"$type,omitempty"` - Avatar *string `json:"avatar,omitempty" cborgen:"avatar"` - CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` -} - -func FeedGetRepostedBy(ctx context.Context, c *xrpc.Client, before string, cid string, limit int64, uri string) (*FeedGetRepostedBy_Output, error) { +// FeedGetRepostedBy calls the XRPC method "app.bsky.feed.getRepostedBy". +// +// cid: If supplied, filters to reposts of specific version (by CID) of the post record. +// uri: Reference (AT-URI) of post record +func FeedGetRepostedBy(ctx context.Context, c lexutil.LexClient, cid string, cursor string, limit int64, uri string) (*FeedGetRepostedBy_Output, error) { var out FeedGetRepostedBy_Output - params := map[string]interface{}{ - "before": before, - "cid": cid, - "limit": limit, - "uri": uri, + params := map[string]interface{}{} + if cid != "" { + params["cid"] = cid + } + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getRepostedBy", params, nil, &out); err != nil { + params["uri"] = uri + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getRepostedBy", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/feedgetSuggestedFeeds.go b/api/bsky/feedgetSuggestedFeeds.go new file mode 100644 index 000000000..2a17f0ade --- /dev/null +++ b/api/bsky/feedgetSuggestedFeeds.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getSuggestedFeeds + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedGetSuggestedFeeds_Output is the output of a app.bsky.feed.getSuggestedFeeds call. +type FeedGetSuggestedFeeds_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feeds []*FeedDefs_GeneratorView `json:"feeds" cborgen:"feeds"` +} + +// FeedGetSuggestedFeeds calls the XRPC method "app.bsky.feed.getSuggestedFeeds". +func FeedGetSuggestedFeeds(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*FeedGetSuggestedFeeds_Output, error) { + var out FeedGetSuggestedFeeds_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getSuggestedFeeds", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetTimeline.go b/api/bsky/feedgetTimeline.go index 3f1cfaa07..637ea7a07 100644 --- a/api/bsky/feedgetTimeline.go +++ b/api/bsky/feedgetTimeline.go @@ -1,31 +1,38 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.getTimeline + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.feed.getTimeline - -func init() { -} - +// FeedGetTimeline_Output is the output of a app.bsky.feed.getTimeline call. type FeedGetTimeline_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Feed []*FeedFeedViewPost `json:"feed" cborgen:"feed"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feed []*FeedDefs_FeedViewPost `json:"feed" cborgen:"feed"` } -func FeedGetTimeline(ctx context.Context, c *xrpc.Client, algorithm string, before string, limit int64) (*FeedGetTimeline_Output, error) { +// FeedGetTimeline calls the XRPC method "app.bsky.feed.getTimeline". +// +// algorithm: Variant 'algorithm' for timeline. Implementation-specific. NOTE: most feed flexibility has been moved to feed generator mechanism. +func FeedGetTimeline(ctx context.Context, c lexutil.LexClient, algorithm string, cursor string, limit int64) (*FeedGetTimeline_Output, error) { var out FeedGetTimeline_Output - params := map[string]interface{}{ - "algorithm": algorithm, - "before": before, - "limit": limit, + params := map[string]interface{}{} + if algorithm != "" { + params["algorithm"] = algorithm + } + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getTimeline", params, nil, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.getTimeline", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/feedgetVotes.go b/api/bsky/feedgetVotes.go deleted file mode 100644 index f3911afe6..000000000 --- a/api/bsky/feedgetVotes.go +++ /dev/null @@ -1,45 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.feed.getVotes - -func init() { -} - -type FeedGetVotes_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid *string `json:"cid,omitempty" cborgen:"cid"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Uri string `json:"uri" cborgen:"uri"` - Votes []*FeedGetVotes_Vote `json:"votes" cborgen:"votes"` -} - -type FeedGetVotes_Vote struct { - LexiconTypeID string `json:"$type,omitempty"` - Actor *ActorRef_WithInfo `json:"actor" cborgen:"actor"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Direction string `json:"direction" cborgen:"direction"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` -} - -func FeedGetVotes(ctx context.Context, c *xrpc.Client, before string, cid string, direction string, limit int64, uri string) (*FeedGetVotes_Output, error) { - var out FeedGetVotes_Output - - params := map[string]interface{}{ - "before": before, - "cid": cid, - "direction": direction, - "limit": limit, - "uri": uri, - } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getVotes", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/feedlike.go b/api/bsky/feedlike.go new file mode 100644 index 000000000..774b092e3 --- /dev/null +++ b/api/bsky/feedlike.go @@ -0,0 +1,21 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.like + +package bsky + +import ( + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.feed.like", &FeedLike{}) +} + +type FeedLike struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.like"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Subject *comatproto.RepoStrongRef `json:"subject" cborgen:"subject"` + Via *comatproto.RepoStrongRef `json:"via,omitempty" cborgen:"via,omitempty"` +} diff --git a/api/bsky/feedpost.go b/api/bsky/feedpost.go index 8c1f5ef66..f3004f25f 100644 --- a/api/bsky/feedpost.go +++ b/api/bsky/feedpost.go @@ -1,4 +1,8 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.post + +package bsky import ( "bytes" @@ -6,30 +10,41 @@ import ( "fmt" "io" - comatprototypes "github.com/bluesky-social/indigo/api/atproto" - "github.com/bluesky-social/indigo/lex/util" + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" cbg "github.com/whyrusleeping/cbor-gen" ) -// schema: app.bsky.feed.post - func init() { - util.RegisterType("app.bsky.feed.post", &FeedPost{}) + lexutil.RegisterType("app.bsky.feed.post", &FeedPost{}) } -// RECORDTYPE: FeedPost type FeedPost struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.post"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Embed *FeedPost_Embed `json:"embed,omitempty" cborgen:"embed"` - Entities []*FeedPost_Entity `json:"entities,omitempty" cborgen:"entities"` - Reply *FeedPost_ReplyRef `json:"reply,omitempty" cborgen:"reply"` - Text string `json:"text" cborgen:"text"` + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.post"` + // createdAt: Client-declared timestamp when this post was originally created. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Embed *FeedPost_Embed `json:"embed,omitempty" cborgen:"embed,omitempty"` + // entities: DEPRECATED: replaced by app.bsky.richtext.facet. + Entities []*FeedPost_Entity `json:"entities,omitempty" cborgen:"entities,omitempty"` + // facets: Annotations of text (mentions, URLs, hashtags, etc) + Facets []*RichtextFacet `json:"facets,omitempty" cborgen:"facets,omitempty"` + // labels: Self-label values for this post. Effectively content warnings. + Labels *FeedPost_Labels `json:"labels,omitempty" cborgen:"labels,omitempty"` + // langs: Indicates human language of post primary text content. + Langs []string `json:"langs,omitempty" cborgen:"langs,omitempty"` + Reply *FeedPost_ReplyRef `json:"reply,omitempty" cborgen:"reply,omitempty"` + // tags: Additional hashtags, in addition to any included in post text and facets. + Tags []string `json:"tags,omitempty" cborgen:"tags,omitempty"` + // text: The primary post content. May be an empty string, if there are embeds. + Text string `json:"text" cborgen:"text"` } type FeedPost_Embed struct { - EmbedImages *EmbedImages - EmbedExternal *EmbedExternal + EmbedImages *EmbedImages + EmbedVideo *EmbedVideo + EmbedExternal *EmbedExternal + EmbedRecord *EmbedRecord + EmbedRecordWithMedia *EmbedRecordWithMedia } func (t *FeedPost_Embed) MarshalJSON() ([]byte, error) { @@ -37,14 +52,27 @@ func (t *FeedPost_Embed) MarshalJSON() ([]byte, error) { t.EmbedImages.LexiconTypeID = "app.bsky.embed.images" return json.Marshal(t.EmbedImages) } + if t.EmbedVideo != nil { + t.EmbedVideo.LexiconTypeID = "app.bsky.embed.video" + return json.Marshal(t.EmbedVideo) + } if t.EmbedExternal != nil { t.EmbedExternal.LexiconTypeID = "app.bsky.embed.external" return json.Marshal(t.EmbedExternal) } - return nil, fmt.Errorf("cannot marshal empty enum") + if t.EmbedRecord != nil { + t.EmbedRecord.LexiconTypeID = "app.bsky.embed.record" + return json.Marshal(t.EmbedRecord) + } + if t.EmbedRecordWithMedia != nil { + t.EmbedRecordWithMedia.LexiconTypeID = "app.bsky.embed.recordWithMedia" + return json.Marshal(t.EmbedRecordWithMedia) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") } + func (t *FeedPost_Embed) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) + typ, err := lexutil.TypeExtract(b) if err != nil { return err } @@ -53,10 +81,18 @@ func (t *FeedPost_Embed) UnmarshalJSON(b []byte) error { case "app.bsky.embed.images": t.EmbedImages = new(EmbedImages) return json.Unmarshal(b, t.EmbedImages) + case "app.bsky.embed.video": + t.EmbedVideo = new(EmbedVideo) + return json.Unmarshal(b, t.EmbedVideo) case "app.bsky.embed.external": t.EmbedExternal = new(EmbedExternal) return json.Unmarshal(b, t.EmbedExternal) - + case "app.bsky.embed.record": + t.EmbedRecord = new(EmbedRecord) + return json.Unmarshal(b, t.EmbedRecord) + case "app.bsky.embed.recordWithMedia": + t.EmbedRecordWithMedia = new(EmbedRecordWithMedia) + return json.Unmarshal(b, t.EmbedRecordWithMedia) default: return nil } @@ -71,13 +107,23 @@ func (t *FeedPost_Embed) MarshalCBOR(w io.Writer) error { if t.EmbedImages != nil { return t.EmbedImages.MarshalCBOR(w) } + if t.EmbedVideo != nil { + return t.EmbedVideo.MarshalCBOR(w) + } if t.EmbedExternal != nil { return t.EmbedExternal.MarshalCBOR(w) } - return fmt.Errorf("cannot cbor marshal empty enum") + if t.EmbedRecord != nil { + return t.EmbedRecord.MarshalCBOR(w) + } + if t.EmbedRecordWithMedia != nil { + return t.EmbedRecordWithMedia.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") } + func (t *FeedPost_Embed) UnmarshalCBOR(r io.Reader) error { - typ, b, err := util.CborTypeExtractReader(r) + typ, b, err := lexutil.CborTypeExtractReader(r) if err != nil { return err } @@ -86,88 +132,98 @@ func (t *FeedPost_Embed) UnmarshalCBOR(r io.Reader) error { case "app.bsky.embed.images": t.EmbedImages = new(EmbedImages) return t.EmbedImages.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.embed.video": + t.EmbedVideo = new(EmbedVideo) + return t.EmbedVideo.UnmarshalCBOR(bytes.NewReader(b)) case "app.bsky.embed.external": t.EmbedExternal = new(EmbedExternal) return t.EmbedExternal.UnmarshalCBOR(bytes.NewReader(b)) - + case "app.bsky.embed.record": + t.EmbedRecord = new(EmbedRecord) + return t.EmbedRecord.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.embed.recordWithMedia": + t.EmbedRecordWithMedia = new(EmbedRecordWithMedia) + return t.EmbedRecordWithMedia.UnmarshalCBOR(bytes.NewReader(b)) default: return nil } } +// FeedPost_Entity is a "entity" in the app.bsky.feed.post schema. +// +// Deprecated: use facets instead. type FeedPost_Entity struct { - LexiconTypeID string `json:"$type,omitempty"` - Index *FeedPost_TextSlice `json:"index" cborgen:"index"` - Type string `json:"type" cborgen:"type"` - Value string `json:"value" cborgen:"value"` + Index *FeedPost_TextSlice `json:"index" cborgen:"index"` + // type: Expected values are 'mention' and 'link'. + Type string `json:"type" cborgen:"type"` + Value string `json:"value" cborgen:"value"` } -type FeedPost_ReplyRef struct { - LexiconTypeID string `json:"$type,omitempty"` - Parent *comatprototypes.RepoStrongRef `json:"parent" cborgen:"parent"` - Root *comatprototypes.RepoStrongRef `json:"root" cborgen:"root"` +// Self-label values for this post. Effectively content warnings. +type FeedPost_Labels struct { + LabelDefs_SelfLabels *comatproto.LabelDefs_SelfLabels } -type FeedPost_TextSlice struct { - LexiconTypeID string `json:"$type,omitempty"` - End int64 `json:"end" cborgen:"end"` - Start int64 `json:"start" cborgen:"start"` +func (t *FeedPost_Labels) MarshalJSON() ([]byte, error) { + if t.LabelDefs_SelfLabels != nil { + t.LabelDefs_SelfLabels.LexiconTypeID = "com.atproto.label.defs#selfLabels" + return json.Marshal(t.LabelDefs_SelfLabels) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") } -type FeedPost_View struct { - LexiconTypeID string `json:"$type,omitempty"` - Author *ActorRef_WithInfo `json:"author" cborgen:"author"` - Cid string `json:"cid" cborgen:"cid"` - DownvoteCount int64 `json:"downvoteCount" cborgen:"downvoteCount"` - Embed *FeedPost_View_Embed `json:"embed,omitempty" cborgen:"embed"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` - Record util.LexiconTypeDecoder `json:"record" cborgen:"record"` - ReplyCount int64 `json:"replyCount" cborgen:"replyCount"` - RepostCount int64 `json:"repostCount" cborgen:"repostCount"` - UpvoteCount int64 `json:"upvoteCount" cborgen:"upvoteCount"` - Uri string `json:"uri" cborgen:"uri"` - Viewer *FeedPost_ViewerState `json:"viewer" cborgen:"viewer"` -} +func (t *FeedPost_Labels) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } -type FeedPost_View_Embed struct { - EmbedImages_Presented *EmbedImages_Presented - EmbedExternal_Presented *EmbedExternal_Presented + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return json.Unmarshal(b, t.LabelDefs_SelfLabels) + default: + return nil + } } -func (t *FeedPost_View_Embed) MarshalJSON() ([]byte, error) { - if t.EmbedImages_Presented != nil { - t.EmbedImages_Presented.LexiconTypeID = "app.bsky.embed.images#presented" - return json.Marshal(t.EmbedImages_Presented) +func (t *FeedPost_Labels) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err } - if t.EmbedExternal_Presented != nil { - t.EmbedExternal_Presented.LexiconTypeID = "app.bsky.embed.external#presented" - return json.Marshal(t.EmbedExternal_Presented) + if t.LabelDefs_SelfLabels != nil { + return t.LabelDefs_SelfLabels.MarshalCBOR(w) } - return nil, fmt.Errorf("cannot marshal empty enum") + return fmt.Errorf("can not marshal empty union as CBOR") } -func (t *FeedPost_View_Embed) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) + +func (t *FeedPost_Labels) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) if err != nil { return err } switch typ { - case "app.bsky.embed.images#presented": - t.EmbedImages_Presented = new(EmbedImages_Presented) - return json.Unmarshal(b, t.EmbedImages_Presented) - case "app.bsky.embed.external#presented": - t.EmbedExternal_Presented = new(EmbedExternal_Presented) - return json.Unmarshal(b, t.EmbedExternal_Presented) - + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return t.LabelDefs_SelfLabels.UnmarshalCBOR(bytes.NewReader(b)) default: return nil } } -type FeedPost_ViewerState struct { - LexiconTypeID string `json:"$type,omitempty"` - Downvote *string `json:"downvote,omitempty" cborgen:"downvote"` - Muted *bool `json:"muted,omitempty" cborgen:"muted"` - Repost *string `json:"repost,omitempty" cborgen:"repost"` - Upvote *string `json:"upvote,omitempty" cborgen:"upvote"` +// FeedPost_ReplyRef is a "replyRef" in the app.bsky.feed.post schema. +type FeedPost_ReplyRef struct { + Parent *comatproto.RepoStrongRef `json:"parent" cborgen:"parent"` + Root *comatproto.RepoStrongRef `json:"root" cborgen:"root"` +} + +// FeedPost_TextSlice is a "textSlice" in the app.bsky.feed.post schema. +// +// Deprecated. Use app.bsky.richtext instead -- A text segment. Start is inclusive, end is exclusive. Indices are for utf16-encoded strings. +type FeedPost_TextSlice struct { + End int64 `json:"end" cborgen:"end"` + Start int64 `json:"start" cborgen:"start"` } diff --git a/api/bsky/feedpostgate.go b/api/bsky/feedpostgate.go new file mode 100644 index 000000000..1bed44431 --- /dev/null +++ b/api/bsky/feedpostgate.go @@ -0,0 +1,91 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.postgate + +package bsky + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func init() { + lexutil.RegisterType("app.bsky.feed.postgate", &FeedPostgate{}) +} + +type FeedPostgate struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.postgate"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // detachedEmbeddingUris: List of AT-URIs embedding this post that the author has detached from. + DetachedEmbeddingUris []string `json:"detachedEmbeddingUris,omitempty" cborgen:"detachedEmbeddingUris,omitempty"` + // embeddingRules: List of rules defining who can embed this post. If value is an empty array or is undefined, no particular rules apply and anyone can embed. + EmbeddingRules []*FeedPostgate_EmbeddingRules_Elem `json:"embeddingRules,omitempty" cborgen:"embeddingRules,omitempty"` + // post: Reference (AT-URI) to the post record. + Post string `json:"post" cborgen:"post"` +} + +// FeedPostgate_DisableRule is a "disableRule" in the app.bsky.feed.postgate schema. +// +// Disables embedding of this post. +type FeedPostgate_DisableRule struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.postgate#disableRule"` +} + +type FeedPostgate_EmbeddingRules_Elem struct { + FeedPostgate_DisableRule *FeedPostgate_DisableRule +} + +func (t *FeedPostgate_EmbeddingRules_Elem) MarshalJSON() ([]byte, error) { + if t.FeedPostgate_DisableRule != nil { + t.FeedPostgate_DisableRule.LexiconTypeID = "app.bsky.feed.postgate#disableRule" + return json.Marshal(t.FeedPostgate_DisableRule) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedPostgate_EmbeddingRules_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.postgate#disableRule": + t.FeedPostgate_DisableRule = new(FeedPostgate_DisableRule) + return json.Unmarshal(b, t.FeedPostgate_DisableRule) + default: + return nil + } +} + +func (t *FeedPostgate_EmbeddingRules_Elem) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.FeedPostgate_DisableRule != nil { + return t.FeedPostgate_DisableRule.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *FeedPostgate_EmbeddingRules_Elem) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.postgate#disableRule": + t.FeedPostgate_DisableRule = new(FeedPostgate_DisableRule) + return t.FeedPostgate_DisableRule.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } +} diff --git a/api/bsky/feedrepost.go b/api/bsky/feedrepost.go index 4f8434060..485eaace0 100644 --- a/api/bsky/feedrepost.go +++ b/api/bsky/feedrepost.go @@ -1,19 +1,21 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.repost + +package bsky import ( - comatprototypes "github.com/bluesky-social/indigo/api/atproto" - "github.com/bluesky-social/indigo/lex/util" + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.feed.repost - func init() { - util.RegisterType("app.bsky.feed.repost", &FeedRepost{}) + lexutil.RegisterType("app.bsky.feed.repost", &FeedRepost{}) } -// RECORDTYPE: FeedRepost type FeedRepost struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.repost"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Subject *comatprototypes.RepoStrongRef `json:"subject" cborgen:"subject"` + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.repost"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Subject *comatproto.RepoStrongRef `json:"subject" cborgen:"subject"` + Via *comatproto.RepoStrongRef `json:"via,omitempty" cborgen:"via,omitempty"` } diff --git a/api/bsky/feedsearchPosts.go b/api/bsky/feedsearchPosts.go new file mode 100644 index 000000000..3777557d7 --- /dev/null +++ b/api/bsky/feedsearchPosts.go @@ -0,0 +1,77 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.searchPosts + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedSearchPosts_Output is the output of a app.bsky.feed.searchPosts call. +type FeedSearchPosts_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // hitsTotal: Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. + HitsTotal *int64 `json:"hitsTotal,omitempty" cborgen:"hitsTotal,omitempty"` + Posts []*FeedDefs_PostView `json:"posts" cborgen:"posts"` +} + +// FeedSearchPosts calls the XRPC method "app.bsky.feed.searchPosts". +// +// author: Filter to posts by the given account. Handles are resolved to DID before query-time. +// cursor: Optional pagination mechanism; may not necessarily allow scrolling through entire result set. +// domain: Filter to posts with URLs (facet links or embeds) linking to the given domain (hostname). Server may apply hostname normalization. +// lang: Filter to posts in the given language. Expected to be based on post language field, though server may override language detection. +// mentions: Filter to posts which mention the given account. Handles are resolved to DID before query-time. Only matches rich-text facet mentions. +// q: Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. +// since: Filter results for posts after the indicated datetime (inclusive). Expected to use 'sortAt' timestamp, which may not match 'createdAt'. Can be a datetime, or just an ISO date (YYYY-MM-DD). +// sort: Specifies the ranking order of results. +// tag: Filter to posts with the given tag (hashtag), based on rich-text facet or tag field. Do not include the hash (#) prefix. Multiple tags can be specified, with 'AND' matching. +// until: Filter results for posts before the indicated datetime (not inclusive). Expected to use 'sortAt' timestamp, which may not match 'createdAt'. Can be a datetime, or just an ISO date (YYY-MM-DD). +// url: Filter to posts with links (facet links or embeds) pointing to this URL. Server may apply URL normalization or fuzzy matching. +func FeedSearchPosts(ctx context.Context, c lexutil.LexClient, author string, cursor string, domain string, lang string, limit int64, mentions string, q string, since string, sort string, tag []string, until string, url string) (*FeedSearchPosts_Output, error) { + var out FeedSearchPosts_Output + + params := map[string]interface{}{} + if author != "" { + params["author"] = author + } + if cursor != "" { + params["cursor"] = cursor + } + if domain != "" { + params["domain"] = domain + } + if lang != "" { + params["lang"] = lang + } + if limit != 0 { + params["limit"] = limit + } + if mentions != "" { + params["mentions"] = mentions + } + params["q"] = q + if since != "" { + params["since"] = since + } + if sort != "" { + params["sort"] = sort + } + if len(tag) != 0 { + params["tag"] = tag + } + if until != "" { + params["until"] = until + } + if url != "" { + params["url"] = url + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.feed.searchPosts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedsendInteractions.go b/api/bsky/feedsendInteractions.go new file mode 100644 index 000000000..d011cb65a --- /dev/null +++ b/api/bsky/feedsendInteractions.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.sendInteractions + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// FeedSendInteractions_Input is the input argument to a app.bsky.feed.sendInteractions call. +type FeedSendInteractions_Input struct { + Interactions []*FeedDefs_Interaction `json:"interactions" cborgen:"interactions"` +} + +// FeedSendInteractions_Output is the output of a app.bsky.feed.sendInteractions call. +type FeedSendInteractions_Output struct { +} + +// FeedSendInteractions calls the XRPC method "app.bsky.feed.sendInteractions". +func FeedSendInteractions(ctx context.Context, c lexutil.LexClient, input *FeedSendInteractions_Input) (*FeedSendInteractions_Output, error) { + var out FeedSendInteractions_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.feed.sendInteractions", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedsetVote.go b/api/bsky/feedsetVote.go deleted file mode 100644 index 543b7853e..000000000 --- a/api/bsky/feedsetVote.go +++ /dev/null @@ -1,34 +0,0 @@ -package schemagen - -import ( - "context" - - comatprototypes "github.com/bluesky-social/indigo/api/atproto" - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.feed.setVote - -func init() { -} - -type FeedSetVote_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - Direction string `json:"direction" cborgen:"direction"` - Subject *comatprototypes.RepoStrongRef `json:"subject" cborgen:"subject"` -} - -type FeedSetVote_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Downvote *string `json:"downvote,omitempty" cborgen:"downvote"` - Upvote *string `json:"upvote,omitempty" cborgen:"upvote"` -} - -func FeedSetVote(ctx context.Context, c *xrpc.Client, input *FeedSetVote_Input) (*FeedSetVote_Output, error) { - var out FeedSetVote_Output - if err := c.Do(ctx, xrpc.Procedure, "application/json", "app.bsky.feed.setVote", nil, input, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/feedthreadgate.go b/api/bsky/feedthreadgate.go new file mode 100644 index 000000000..064e04285 --- /dev/null +++ b/api/bsky/feedthreadgate.go @@ -0,0 +1,155 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.feed.threadgate + +package bsky + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func init() { + lexutil.RegisterType("app.bsky.feed.threadgate", &FeedThreadgate{}) +} + +type FeedThreadgate struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.threadgate"` + // allow: List of rules defining who can reply to this post. If value is an empty array, no one can reply. If value is undefined, anyone can reply. + Allow []*FeedThreadgate_Allow_Elem `json:"allow,omitempty" cborgen:"allow,omitempty"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // hiddenReplies: List of hidden reply URIs. + HiddenReplies []string `json:"hiddenReplies,omitempty" cborgen:"hiddenReplies,omitempty"` + // post: Reference (AT-URI) to the post record. + Post string `json:"post" cborgen:"post"` +} + +type FeedThreadgate_Allow_Elem struct { + FeedThreadgate_MentionRule *FeedThreadgate_MentionRule + FeedThreadgate_FollowerRule *FeedThreadgate_FollowerRule + FeedThreadgate_FollowingRule *FeedThreadgate_FollowingRule + FeedThreadgate_ListRule *FeedThreadgate_ListRule +} + +func (t *FeedThreadgate_Allow_Elem) MarshalJSON() ([]byte, error) { + if t.FeedThreadgate_MentionRule != nil { + t.FeedThreadgate_MentionRule.LexiconTypeID = "app.bsky.feed.threadgate#mentionRule" + return json.Marshal(t.FeedThreadgate_MentionRule) + } + if t.FeedThreadgate_FollowerRule != nil { + t.FeedThreadgate_FollowerRule.LexiconTypeID = "app.bsky.feed.threadgate#followerRule" + return json.Marshal(t.FeedThreadgate_FollowerRule) + } + if t.FeedThreadgate_FollowingRule != nil { + t.FeedThreadgate_FollowingRule.LexiconTypeID = "app.bsky.feed.threadgate#followingRule" + return json.Marshal(t.FeedThreadgate_FollowingRule) + } + if t.FeedThreadgate_ListRule != nil { + t.FeedThreadgate_ListRule.LexiconTypeID = "app.bsky.feed.threadgate#listRule" + return json.Marshal(t.FeedThreadgate_ListRule) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *FeedThreadgate_Allow_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.threadgate#mentionRule": + t.FeedThreadgate_MentionRule = new(FeedThreadgate_MentionRule) + return json.Unmarshal(b, t.FeedThreadgate_MentionRule) + case "app.bsky.feed.threadgate#followerRule": + t.FeedThreadgate_FollowerRule = new(FeedThreadgate_FollowerRule) + return json.Unmarshal(b, t.FeedThreadgate_FollowerRule) + case "app.bsky.feed.threadgate#followingRule": + t.FeedThreadgate_FollowingRule = new(FeedThreadgate_FollowingRule) + return json.Unmarshal(b, t.FeedThreadgate_FollowingRule) + case "app.bsky.feed.threadgate#listRule": + t.FeedThreadgate_ListRule = new(FeedThreadgate_ListRule) + return json.Unmarshal(b, t.FeedThreadgate_ListRule) + default: + return nil + } +} + +func (t *FeedThreadgate_Allow_Elem) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.FeedThreadgate_MentionRule != nil { + return t.FeedThreadgate_MentionRule.MarshalCBOR(w) + } + if t.FeedThreadgate_FollowerRule != nil { + return t.FeedThreadgate_FollowerRule.MarshalCBOR(w) + } + if t.FeedThreadgate_FollowingRule != nil { + return t.FeedThreadgate_FollowingRule.MarshalCBOR(w) + } + if t.FeedThreadgate_ListRule != nil { + return t.FeedThreadgate_ListRule.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *FeedThreadgate_Allow_Elem) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.threadgate#mentionRule": + t.FeedThreadgate_MentionRule = new(FeedThreadgate_MentionRule) + return t.FeedThreadgate_MentionRule.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.feed.threadgate#followerRule": + t.FeedThreadgate_FollowerRule = new(FeedThreadgate_FollowerRule) + return t.FeedThreadgate_FollowerRule.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.feed.threadgate#followingRule": + t.FeedThreadgate_FollowingRule = new(FeedThreadgate_FollowingRule) + return t.FeedThreadgate_FollowingRule.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.feed.threadgate#listRule": + t.FeedThreadgate_ListRule = new(FeedThreadgate_ListRule) + return t.FeedThreadgate_ListRule.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } +} + +// FeedThreadgate_FollowerRule is a "followerRule" in the app.bsky.feed.threadgate schema. +// +// Allow replies from actors who follow you. +type FeedThreadgate_FollowerRule struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.threadgate#followerRule"` +} + +// FeedThreadgate_FollowingRule is a "followingRule" in the app.bsky.feed.threadgate schema. +// +// Allow replies from actors you follow. +type FeedThreadgate_FollowingRule struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.threadgate#followingRule"` +} + +// FeedThreadgate_ListRule is a "listRule" in the app.bsky.feed.threadgate schema. +// +// Allow replies from actors on a list. +type FeedThreadgate_ListRule struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.threadgate#listRule"` + List string `json:"list" cborgen:"list"` +} + +// FeedThreadgate_MentionRule is a "mentionRule" in the app.bsky.feed.threadgate schema. +// +// Allow replies from actors mentioned in your post. +type FeedThreadgate_MentionRule struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.threadgate#mentionRule"` +} diff --git a/api/bsky/feedtrend.go b/api/bsky/feedtrend.go deleted file mode 100644 index a58beaa70..000000000 --- a/api/bsky/feedtrend.go +++ /dev/null @@ -1,19 +0,0 @@ -package schemagen - -import ( - comatprototypes "github.com/bluesky-social/indigo/api/atproto" - "github.com/bluesky-social/indigo/lex/util" -) - -// schema: app.bsky.feed.trend - -func init() { - util.RegisterType("app.bsky.feed.trend", &FeedTrend{}) -} - -// RECORDTYPE: FeedTrend -type FeedTrend struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.trend"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Subject *comatprototypes.RepoStrongRef `json:"subject" cborgen:"subject"` -} diff --git a/api/bsky/feedvote.go b/api/bsky/feedvote.go deleted file mode 100644 index 71acf2d4b..000000000 --- a/api/bsky/feedvote.go +++ /dev/null @@ -1,20 +0,0 @@ -package schemagen - -import ( - comatprototypes "github.com/bluesky-social/indigo/api/atproto" - "github.com/bluesky-social/indigo/lex/util" -) - -// schema: app.bsky.feed.vote - -func init() { - util.RegisterType("app.bsky.feed.vote", &FeedVote{}) -} - -// RECORDTYPE: FeedVote -type FeedVote struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.feed.vote"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Direction string `json:"direction" cborgen:"direction"` - Subject *comatprototypes.RepoStrongRef `json:"subject" cborgen:"subject"` -} diff --git a/api/bsky/graphassertCreator.go b/api/bsky/graphassertCreator.go deleted file mode 100644 index b4f96cfc4..000000000 --- a/api/bsky/graphassertCreator.go +++ /dev/null @@ -1,8 +0,0 @@ -package schemagen - -// schema: app.bsky.graph.assertCreator - -func init() { -} - -const GraphAssertCreator = "app.bsky.graph.assertCreator" diff --git a/api/bsky/graphassertMember.go b/api/bsky/graphassertMember.go deleted file mode 100644 index 103a926fd..000000000 --- a/api/bsky/graphassertMember.go +++ /dev/null @@ -1,8 +0,0 @@ -package schemagen - -// schema: app.bsky.graph.assertMember - -func init() { -} - -const GraphAssertMember = "app.bsky.graph.assertMember" diff --git a/api/bsky/graphassertion.go b/api/bsky/graphassertion.go deleted file mode 100644 index 21b541336..000000000 --- a/api/bsky/graphassertion.go +++ /dev/null @@ -1,19 +0,0 @@ -package schemagen - -import ( - "github.com/bluesky-social/indigo/lex/util" -) - -// schema: app.bsky.graph.assertion - -func init() { - util.RegisterType("app.bsky.graph.assertion", &GraphAssertion{}) -} - -// RECORDTYPE: GraphAssertion -type GraphAssertion struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.assertion"` - Assertion string `json:"assertion" cborgen:"assertion"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Subject *ActorRef `json:"subject" cborgen:"subject"` -} diff --git a/api/bsky/graphblock.go b/api/bsky/graphblock.go new file mode 100644 index 000000000..ddd4a7d26 --- /dev/null +++ b/api/bsky/graphblock.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.block + +package bsky + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.graph.block", &GraphBlock{}) +} + +type GraphBlock struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.block"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // subject: DID of the account to be blocked. + Subject string `json:"subject" cborgen:"subject"` +} diff --git a/api/bsky/graphconfirmation.go b/api/bsky/graphconfirmation.go deleted file mode 100644 index fb738df81..000000000 --- a/api/bsky/graphconfirmation.go +++ /dev/null @@ -1,20 +0,0 @@ -package schemagen - -import ( - comatprototypes "github.com/bluesky-social/indigo/api/atproto" - "github.com/bluesky-social/indigo/lex/util" -) - -// schema: app.bsky.graph.confirmation - -func init() { - util.RegisterType("app.bsky.graph.confirmation", &GraphConfirmation{}) -} - -// RECORDTYPE: GraphConfirmation -type GraphConfirmation struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.confirmation"` - Assertion *comatprototypes.RepoStrongRef `json:"assertion" cborgen:"assertion"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Originator *ActorRef `json:"originator" cborgen:"originator"` -} diff --git a/api/bsky/graphdefs.go b/api/bsky/graphdefs.go new file mode 100644 index 000000000..a5df693eb --- /dev/null +++ b/api/bsky/graphdefs.go @@ -0,0 +1,110 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.defs + +package bsky + +import ( + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphDefs_ListItemView is a "listItemView" in the app.bsky.graph.defs schema. +type GraphDefs_ListItemView struct { + Subject *ActorDefs_ProfileView `json:"subject" cborgen:"subject"` + Uri string `json:"uri" cborgen:"uri"` +} + +// GraphDefs_ListView is a "listView" in the app.bsky.graph.defs schema. +type GraphDefs_ListView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.defs#listView"` + Avatar *string `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + Cid string `json:"cid" cborgen:"cid"` + Creator *ActorDefs_ProfileView `json:"creator" cborgen:"creator"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + DescriptionFacets []*RichtextFacet `json:"descriptionFacets,omitempty" cborgen:"descriptionFacets,omitempty"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + ListItemCount *int64 `json:"listItemCount,omitempty" cborgen:"listItemCount,omitempty"` + Name string `json:"name" cborgen:"name"` + Purpose *string `json:"purpose" cborgen:"purpose"` + Uri string `json:"uri" cborgen:"uri"` + Viewer *GraphDefs_ListViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// GraphDefs_ListViewBasic is a "listViewBasic" in the app.bsky.graph.defs schema. +type GraphDefs_ListViewBasic struct { + Avatar *string `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + Cid string `json:"cid" cborgen:"cid"` + IndexedAt *string `json:"indexedAt,omitempty" cborgen:"indexedAt,omitempty"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + ListItemCount *int64 `json:"listItemCount,omitempty" cborgen:"listItemCount,omitempty"` + Name string `json:"name" cborgen:"name"` + Purpose *string `json:"purpose" cborgen:"purpose"` + Uri string `json:"uri" cborgen:"uri"` + Viewer *GraphDefs_ListViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// GraphDefs_ListViewerState is a "listViewerState" in the app.bsky.graph.defs schema. +type GraphDefs_ListViewerState struct { + Blocked *string `json:"blocked,omitempty" cborgen:"blocked,omitempty"` + Muted *bool `json:"muted,omitempty" cborgen:"muted,omitempty"` +} + +// GraphDefs_NotFoundActor is a "notFoundActor" in the app.bsky.graph.defs schema. +// +// indicates that a handle or DID could not be resolved +type GraphDefs_NotFoundActor struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.defs#notFoundActor"` + Actor string `json:"actor" cborgen:"actor"` + NotFound bool `json:"notFound" cborgen:"notFound"` +} + +// GraphDefs_Relationship is a "relationship" in the app.bsky.graph.defs schema. +// +// lists the bi-directional graph relationships between one actor (not indicated in the object), and the target actors (the DID included in the object) +type GraphDefs_Relationship struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.defs#relationship"` + // blockedBy: if the actor is blocked by this DID, contains the AT-URI of the block record + BlockedBy *string `json:"blockedBy,omitempty" cborgen:"blockedBy,omitempty"` + // blockedByList: if the actor is blocked by this DID via a block list, contains the AT-URI of the listblock record + BlockedByList *string `json:"blockedByList,omitempty" cborgen:"blockedByList,omitempty"` + // blocking: if the actor blocks this DID, this is the AT-URI of the block record + Blocking *string `json:"blocking,omitempty" cborgen:"blocking,omitempty"` + // blockingByList: if the actor blocks this DID via a block list, this is the AT-URI of the listblock record + BlockingByList *string `json:"blockingByList,omitempty" cborgen:"blockingByList,omitempty"` + Did string `json:"did" cborgen:"did"` + // followedBy: if the actor is followed by this DID, contains the AT-URI of the follow record + FollowedBy *string `json:"followedBy,omitempty" cborgen:"followedBy,omitempty"` + // following: if the actor follows this DID, this is the AT-URI of the follow record + Following *string `json:"following,omitempty" cborgen:"following,omitempty"` +} + +// GraphDefs_StarterPackView is a "starterPackView" in the app.bsky.graph.defs schema. +type GraphDefs_StarterPackView struct { + Cid string `json:"cid" cborgen:"cid"` + Creator *ActorDefs_ProfileViewBasic `json:"creator" cborgen:"creator"` + Feeds []*FeedDefs_GeneratorView `json:"feeds,omitempty" cborgen:"feeds,omitempty"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + JoinedAllTimeCount *int64 `json:"joinedAllTimeCount,omitempty" cborgen:"joinedAllTimeCount,omitempty"` + JoinedWeekCount *int64 `json:"joinedWeekCount,omitempty" cborgen:"joinedWeekCount,omitempty"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + List *GraphDefs_ListViewBasic `json:"list,omitempty" cborgen:"list,omitempty"` + ListItemsSample []*GraphDefs_ListItemView `json:"listItemsSample,omitempty" cborgen:"listItemsSample,omitempty"` + Record *lexutil.LexiconTypeDecoder `json:"record" cborgen:"record"` + Uri string `json:"uri" cborgen:"uri"` +} + +// GraphDefs_StarterPackViewBasic is a "starterPackViewBasic" in the app.bsky.graph.defs schema. +type GraphDefs_StarterPackViewBasic struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.defs#starterPackViewBasic"` + Cid string `json:"cid" cborgen:"cid"` + Creator *ActorDefs_ProfileViewBasic `json:"creator" cborgen:"creator"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + JoinedAllTimeCount *int64 `json:"joinedAllTimeCount,omitempty" cborgen:"joinedAllTimeCount,omitempty"` + JoinedWeekCount *int64 `json:"joinedWeekCount,omitempty" cborgen:"joinedWeekCount,omitempty"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + ListItemCount *int64 `json:"listItemCount,omitempty" cborgen:"listItemCount,omitempty"` + Record *lexutil.LexiconTypeDecoder `json:"record" cborgen:"record"` + Uri string `json:"uri" cborgen:"uri"` +} diff --git a/api/bsky/graphfollow.go b/api/bsky/graphfollow.go index aaff80d3f..7d49155aa 100644 --- a/api/bsky/graphfollow.go +++ b/api/bsky/graphfollow.go @@ -1,18 +1,21 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.follow + +package bsky import ( - "github.com/bluesky-social/indigo/lex/util" + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.graph.follow - func init() { - util.RegisterType("app.bsky.graph.follow", &GraphFollow{}) + lexutil.RegisterType("app.bsky.graph.follow", &GraphFollow{}) } -// RECORDTYPE: GraphFollow type GraphFollow struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.follow"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Subject *ActorRef `json:"subject" cborgen:"subject"` + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.follow"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Subject string `json:"subject" cborgen:"subject"` + Via *comatproto.RepoStrongRef `json:"via,omitempty" cborgen:"via,omitempty"` } diff --git a/api/bsky/graphgetActorStarterPacks.go b/api/bsky/graphgetActorStarterPacks.go new file mode 100644 index 000000000..e1bd6575e --- /dev/null +++ b/api/bsky/graphgetActorStarterPacks.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getActorStarterPacks + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetActorStarterPacks_Output is the output of a app.bsky.graph.getActorStarterPacks call. +type GraphGetActorStarterPacks_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + StarterPacks []*GraphDefs_StarterPackViewBasic `json:"starterPacks" cborgen:"starterPacks"` +} + +// GraphGetActorStarterPacks calls the XRPC method "app.bsky.graph.getActorStarterPacks". +func GraphGetActorStarterPacks(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64) (*GraphGetActorStarterPacks_Output, error) { + var out GraphGetActorStarterPacks_Output + + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getActorStarterPacks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetAssertions.go b/api/bsky/graphgetAssertions.go deleted file mode 100644 index 234120f7a..000000000 --- a/api/bsky/graphgetAssertions.go +++ /dev/null @@ -1,56 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.graph.getAssertions - -func init() { -} - -type GraphGetAssertions_Assertion struct { - LexiconTypeID string `json:"$type,omitempty"` - Assertion string `json:"assertion" cborgen:"assertion"` - Author *ActorRef_WithInfo `json:"author" cborgen:"author"` - Cid string `json:"cid" cborgen:"cid"` - Confirmation *GraphGetAssertions_Confirmation `json:"confirmation,omitempty" cborgen:"confirmation"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` - Subject *ActorRef_WithInfo `json:"subject" cborgen:"subject"` - Uri string `json:"uri" cborgen:"uri"` -} - -type GraphGetAssertions_Confirmation struct { - LexiconTypeID string `json:"$type,omitempty"` - Cid string `json:"cid" cborgen:"cid"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` - Uri string `json:"uri" cborgen:"uri"` -} - -type GraphGetAssertions_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Assertions []*GraphGetAssertions_Assertion `json:"assertions" cborgen:"assertions"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` -} - -func GraphGetAssertions(ctx context.Context, c *xrpc.Client, assertion string, author string, before string, confirmed bool, limit int64, subject string) (*GraphGetAssertions_Output, error) { - var out GraphGetAssertions_Output - - params := map[string]interface{}{ - "assertion": assertion, - "author": author, - "before": before, - "confirmed": confirmed, - "limit": limit, - "subject": subject, - } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.getAssertions", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/graphgetBlocks.go b/api/bsky/graphgetBlocks.go new file mode 100644 index 000000000..d1f70929d --- /dev/null +++ b/api/bsky/graphgetBlocks.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getBlocks + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetBlocks_Output is the output of a app.bsky.graph.getBlocks call. +type GraphGetBlocks_Output struct { + Blocks []*ActorDefs_ProfileView `json:"blocks" cborgen:"blocks"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// GraphGetBlocks calls the XRPC method "app.bsky.graph.getBlocks". +func GraphGetBlocks(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*GraphGetBlocks_Output, error) { + var out GraphGetBlocks_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getBlocks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetFollowers.go b/api/bsky/graphgetFollowers.go index 4943e0bd0..245459087 100644 --- a/api/bsky/graphgetFollowers.go +++ b/api/bsky/graphgetFollowers.go @@ -1,43 +1,35 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getFollowers + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.graph.getFollowers - -func init() { -} - -type GraphGetFollowers_Follower struct { - LexiconTypeID string `json:"$type,omitempty"` - Avatar *string `json:"avatar,omitempty" cborgen:"avatar"` - CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` -} - +// GraphGetFollowers_Output is the output of a app.bsky.graph.getFollowers call. type GraphGetFollowers_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Followers []*GraphGetFollowers_Follower `json:"followers" cborgen:"followers"` - Subject *ActorRef_WithInfo `json:"subject" cborgen:"subject"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Followers []*ActorDefs_ProfileView `json:"followers" cborgen:"followers"` + Subject *ActorDefs_ProfileView `json:"subject" cborgen:"subject"` } -func GraphGetFollowers(ctx context.Context, c *xrpc.Client, before string, limit int64, user string) (*GraphGetFollowers_Output, error) { +// GraphGetFollowers calls the XRPC method "app.bsky.graph.getFollowers". +func GraphGetFollowers(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64) (*GraphGetFollowers_Output, error) { var out GraphGetFollowers_Output - params := map[string]interface{}{ - "before": before, - "limit": limit, - "user": user, + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.getFollowers", params, nil, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getFollowers", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/graphgetFollows.go b/api/bsky/graphgetFollows.go index 5a094bbf1..2ab0fb3ea 100644 --- a/api/bsky/graphgetFollows.go +++ b/api/bsky/graphgetFollows.go @@ -1,42 +1,35 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getFollows + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.graph.getFollows - -func init() { -} - -type GraphGetFollows_Follow struct { - LexiconTypeID string `json:"$type,omitempty"` - CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` -} - +// GraphGetFollows_Output is the output of a app.bsky.graph.getFollows call. type GraphGetFollows_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Follows []*GraphGetFollows_Follow `json:"follows" cborgen:"follows"` - Subject *ActorRef_WithInfo `json:"subject" cborgen:"subject"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Follows []*ActorDefs_ProfileView `json:"follows" cborgen:"follows"` + Subject *ActorDefs_ProfileView `json:"subject" cborgen:"subject"` } -func GraphGetFollows(ctx context.Context, c *xrpc.Client, before string, limit int64, user string) (*GraphGetFollows_Output, error) { +// GraphGetFollows calls the XRPC method "app.bsky.graph.getFollows". +func GraphGetFollows(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64) (*GraphGetFollows_Output, error) { var out GraphGetFollows_Output - params := map[string]interface{}{ - "before": before, - "limit": limit, - "user": user, + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.getFollows", params, nil, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getFollows", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/graphgetKnownFollowers.go b/api/bsky/graphgetKnownFollowers.go new file mode 100644 index 000000000..7c4c9db96 --- /dev/null +++ b/api/bsky/graphgetKnownFollowers.go @@ -0,0 +1,37 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getKnownFollowers + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetKnownFollowers_Output is the output of a app.bsky.graph.getKnownFollowers call. +type GraphGetKnownFollowers_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Followers []*ActorDefs_ProfileView `json:"followers" cborgen:"followers"` + Subject *ActorDefs_ProfileView `json:"subject" cborgen:"subject"` +} + +// GraphGetKnownFollowers calls the XRPC method "app.bsky.graph.getKnownFollowers". +func GraphGetKnownFollowers(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64) (*GraphGetKnownFollowers_Output, error) { + var out GraphGetKnownFollowers_Output + + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getKnownFollowers", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetList.go b/api/bsky/graphgetList.go new file mode 100644 index 000000000..69d9891b9 --- /dev/null +++ b/api/bsky/graphgetList.go @@ -0,0 +1,39 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getList + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetList_Output is the output of a app.bsky.graph.getList call. +type GraphGetList_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Items []*GraphDefs_ListItemView `json:"items" cborgen:"items"` + List *GraphDefs_ListView `json:"list" cborgen:"list"` +} + +// GraphGetList calls the XRPC method "app.bsky.graph.getList". +// +// list: Reference (AT-URI) of the list record to hydrate. +func GraphGetList(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, list string) (*GraphGetList_Output, error) { + var out GraphGetList_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["list"] = list + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getList", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetListBlocks.go b/api/bsky/graphgetListBlocks.go new file mode 100644 index 000000000..edf1423be --- /dev/null +++ b/api/bsky/graphgetListBlocks.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getListBlocks + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetListBlocks_Output is the output of a app.bsky.graph.getListBlocks call. +type GraphGetListBlocks_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Lists []*GraphDefs_ListView `json:"lists" cborgen:"lists"` +} + +// GraphGetListBlocks calls the XRPC method "app.bsky.graph.getListBlocks". +func GraphGetListBlocks(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*GraphGetListBlocks_Output, error) { + var out GraphGetListBlocks_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getListBlocks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetListMutes.go b/api/bsky/graphgetListMutes.go new file mode 100644 index 000000000..03eb976f0 --- /dev/null +++ b/api/bsky/graphgetListMutes.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getListMutes + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetListMutes_Output is the output of a app.bsky.graph.getListMutes call. +type GraphGetListMutes_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Lists []*GraphDefs_ListView `json:"lists" cborgen:"lists"` +} + +// GraphGetListMutes calls the XRPC method "app.bsky.graph.getListMutes". +func GraphGetListMutes(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*GraphGetListMutes_Output, error) { + var out GraphGetListMutes_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getListMutes", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetLists.go b/api/bsky/graphgetLists.go new file mode 100644 index 000000000..1ca8df35a --- /dev/null +++ b/api/bsky/graphgetLists.go @@ -0,0 +1,42 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getLists + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetLists_Output is the output of a app.bsky.graph.getLists call. +type GraphGetLists_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Lists []*GraphDefs_ListView `json:"lists" cborgen:"lists"` +} + +// GraphGetLists calls the XRPC method "app.bsky.graph.getLists". +// +// actor: The account (actor) to enumerate lists from. +// purposes: Optional filter by list purpose. If not specified, all supported types are returned. +func GraphGetLists(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64, purposes []string) (*GraphGetLists_Output, error) { + var out GraphGetLists_Output + + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if len(purposes) != 0 { + params["purposes"] = purposes + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getLists", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetListsWithMembership.go b/api/bsky/graphgetListsWithMembership.go new file mode 100644 index 000000000..4b829752d --- /dev/null +++ b/api/bsky/graphgetListsWithMembership.go @@ -0,0 +1,50 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getListsWithMembership + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetListsWithMembership_ListWithMembership is a "listWithMembership" in the app.bsky.graph.getListsWithMembership schema. +// +// A list and an optional list item indicating membership of a target user to that list. +type GraphGetListsWithMembership_ListWithMembership struct { + List *GraphDefs_ListView `json:"list" cborgen:"list"` + ListItem *GraphDefs_ListItemView `json:"listItem,omitempty" cborgen:"listItem,omitempty"` +} + +// GraphGetListsWithMembership_Output is the output of a app.bsky.graph.getListsWithMembership call. +type GraphGetListsWithMembership_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + ListsWithMembership []*GraphGetListsWithMembership_ListWithMembership `json:"listsWithMembership" cborgen:"listsWithMembership"` +} + +// GraphGetListsWithMembership calls the XRPC method "app.bsky.graph.getListsWithMembership". +// +// actor: The account (actor) to check for membership. +// purposes: Optional filter by list purpose. If not specified, all supported types are returned. +func GraphGetListsWithMembership(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64, purposes []string) (*GraphGetListsWithMembership_Output, error) { + var out GraphGetListsWithMembership_Output + + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if len(purposes) != 0 { + params["purposes"] = purposes + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getListsWithMembership", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetMembers.go b/api/bsky/graphgetMembers.go deleted file mode 100644 index 161326400..000000000 --- a/api/bsky/graphgetMembers.go +++ /dev/null @@ -1,44 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.graph.getMembers - -func init() { -} - -type GraphGetMembers_Member struct { - LexiconTypeID string `json:"$type,omitempty"` - CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` -} - -type GraphGetMembers_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Members []*GraphGetMembers_Member `json:"members" cborgen:"members"` - Subject *ActorRef_WithInfo `json:"subject" cborgen:"subject"` -} - -func GraphGetMembers(ctx context.Context, c *xrpc.Client, actor string, before string, limit int64) (*GraphGetMembers_Output, error) { - var out GraphGetMembers_Output - - params := map[string]interface{}{ - "actor": actor, - "before": before, - "limit": limit, - } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.getMembers", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/graphgetMemberships.go b/api/bsky/graphgetMemberships.go deleted file mode 100644 index 3fe29fbd9..000000000 --- a/api/bsky/graphgetMemberships.go +++ /dev/null @@ -1,44 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.graph.getMemberships - -func init() { -} - -type GraphGetMemberships_Membership struct { - LexiconTypeID string `json:"$type,omitempty"` - CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` -} - -type GraphGetMemberships_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Memberships []*GraphGetMemberships_Membership `json:"memberships" cborgen:"memberships"` - Subject *ActorRef_WithInfo `json:"subject" cborgen:"subject"` -} - -func GraphGetMemberships(ctx context.Context, c *xrpc.Client, actor string, before string, limit int64) (*GraphGetMemberships_Output, error) { - var out GraphGetMemberships_Output - - params := map[string]interface{}{ - "actor": actor, - "before": before, - "limit": limit, - } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.getMemberships", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/graphgetMutes.go b/api/bsky/graphgetMutes.go index 4ab710e11..32f4e6292 100644 --- a/api/bsky/graphgetMutes.go +++ b/api/bsky/graphgetMutes.go @@ -1,39 +1,33 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getMutes + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.graph.getMutes - -func init() { -} - -type GraphGetMutes_Mute struct { - LexiconTypeID string `json:"$type,omitempty"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Declaration *SystemDeclRef `json:"declaration" cborgen:"declaration"` - Did string `json:"did" cborgen:"did"` - DisplayName *string `json:"displayName,omitempty" cborgen:"displayName"` - Handle string `json:"handle" cborgen:"handle"` -} - +// GraphGetMutes_Output is the output of a app.bsky.graph.getMutes call. type GraphGetMutes_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Mutes []*GraphGetMutes_Mute `json:"mutes" cborgen:"mutes"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Mutes []*ActorDefs_ProfileView `json:"mutes" cborgen:"mutes"` } -func GraphGetMutes(ctx context.Context, c *xrpc.Client, before string, limit int64) (*GraphGetMutes_Output, error) { +// GraphGetMutes calls the XRPC method "app.bsky.graph.getMutes". +func GraphGetMutes(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*GraphGetMutes_Output, error) { var out GraphGetMutes_Output - params := map[string]interface{}{ - "before": before, - "limit": limit, + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.getMutes", params, nil, &out); err != nil { + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getMutes", params, nil, &out); err != nil { return nil, err } diff --git a/api/bsky/graphgetRelationships.go b/api/bsky/graphgetRelationships.go new file mode 100644 index 000000000..2cc6edc60 --- /dev/null +++ b/api/bsky/graphgetRelationships.go @@ -0,0 +1,73 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getRelationships + +package bsky + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetRelationships_Output is the output of a app.bsky.graph.getRelationships call. +type GraphGetRelationships_Output struct { + Actor *string `json:"actor,omitempty" cborgen:"actor,omitempty"` + Relationships []*GraphGetRelationships_Output_Relationships_Elem `json:"relationships" cborgen:"relationships"` +} + +type GraphGetRelationships_Output_Relationships_Elem struct { + GraphDefs_Relationship *GraphDefs_Relationship + GraphDefs_NotFoundActor *GraphDefs_NotFoundActor +} + +func (t *GraphGetRelationships_Output_Relationships_Elem) MarshalJSON() ([]byte, error) { + if t.GraphDefs_Relationship != nil { + t.GraphDefs_Relationship.LexiconTypeID = "app.bsky.graph.defs#relationship" + return json.Marshal(t.GraphDefs_Relationship) + } + if t.GraphDefs_NotFoundActor != nil { + t.GraphDefs_NotFoundActor.LexiconTypeID = "app.bsky.graph.defs#notFoundActor" + return json.Marshal(t.GraphDefs_NotFoundActor) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *GraphGetRelationships_Output_Relationships_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.graph.defs#relationship": + t.GraphDefs_Relationship = new(GraphDefs_Relationship) + return json.Unmarshal(b, t.GraphDefs_Relationship) + case "app.bsky.graph.defs#notFoundActor": + t.GraphDefs_NotFoundActor = new(GraphDefs_NotFoundActor) + return json.Unmarshal(b, t.GraphDefs_NotFoundActor) + default: + return nil + } +} + +// GraphGetRelationships calls the XRPC method "app.bsky.graph.getRelationships". +// +// actor: Primary account requesting relationships for. +// others: List of 'other' accounts to be related back to the primary. +func GraphGetRelationships(ctx context.Context, c lexutil.LexClient, actor string, others []string) (*GraphGetRelationships_Output, error) { + var out GraphGetRelationships_Output + + params := map[string]interface{}{} + params["actor"] = actor + if len(others) != 0 { + params["others"] = others + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getRelationships", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetStarterPack.go b/api/bsky/graphgetStarterPack.go new file mode 100644 index 000000000..6652ef128 --- /dev/null +++ b/api/bsky/graphgetStarterPack.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getStarterPack + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetStarterPack_Output is the output of a app.bsky.graph.getStarterPack call. +type GraphGetStarterPack_Output struct { + StarterPack *GraphDefs_StarterPackView `json:"starterPack" cborgen:"starterPack"` +} + +// GraphGetStarterPack calls the XRPC method "app.bsky.graph.getStarterPack". +// +// starterPack: Reference (AT-URI) of the starter pack record. +func GraphGetStarterPack(ctx context.Context, c lexutil.LexClient, starterPack string) (*GraphGetStarterPack_Output, error) { + var out GraphGetStarterPack_Output + + params := map[string]interface{}{} + params["starterPack"] = starterPack + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getStarterPack", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetStarterPacks.go b/api/bsky/graphgetStarterPacks.go new file mode 100644 index 000000000..c4475776f --- /dev/null +++ b/api/bsky/graphgetStarterPacks.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getStarterPacks + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetStarterPacks_Output is the output of a app.bsky.graph.getStarterPacks call. +type GraphGetStarterPacks_Output struct { + StarterPacks []*GraphDefs_StarterPackViewBasic `json:"starterPacks" cborgen:"starterPacks"` +} + +// GraphGetStarterPacks calls the XRPC method "app.bsky.graph.getStarterPacks". +func GraphGetStarterPacks(ctx context.Context, c lexutil.LexClient, uris []string) (*GraphGetStarterPacks_Output, error) { + var out GraphGetStarterPacks_Output + + params := map[string]interface{}{} + params["uris"] = uris + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getStarterPacks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetStarterPacksWithMembership.go b/api/bsky/graphgetStarterPacksWithMembership.go new file mode 100644 index 000000000..a738b4601 --- /dev/null +++ b/api/bsky/graphgetStarterPacksWithMembership.go @@ -0,0 +1,46 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getStarterPacksWithMembership + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetStarterPacksWithMembership_Output is the output of a app.bsky.graph.getStarterPacksWithMembership call. +type GraphGetStarterPacksWithMembership_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + StarterPacksWithMembership []*GraphGetStarterPacksWithMembership_StarterPackWithMembership `json:"starterPacksWithMembership" cborgen:"starterPacksWithMembership"` +} + +// GraphGetStarterPacksWithMembership_StarterPackWithMembership is a "starterPackWithMembership" in the app.bsky.graph.getStarterPacksWithMembership schema. +// +// A starter pack and an optional list item indicating membership of a target user to that starter pack. +type GraphGetStarterPacksWithMembership_StarterPackWithMembership struct { + ListItem *GraphDefs_ListItemView `json:"listItem,omitempty" cborgen:"listItem,omitempty"` + StarterPack *GraphDefs_StarterPackView `json:"starterPack" cborgen:"starterPack"` +} + +// GraphGetStarterPacksWithMembership calls the XRPC method "app.bsky.graph.getStarterPacksWithMembership". +// +// actor: The account (actor) to check for membership. +func GraphGetStarterPacksWithMembership(ctx context.Context, c lexutil.LexClient, actor string, cursor string, limit int64) (*GraphGetStarterPacksWithMembership_Output, error) { + var out GraphGetStarterPacksWithMembership_Output + + params := map[string]interface{}{} + params["actor"] = actor + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getStarterPacksWithMembership", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphgetSuggestedFollowsByActor.go b/api/bsky/graphgetSuggestedFollowsByActor.go new file mode 100644 index 000000000..ec1b8269b --- /dev/null +++ b/api/bsky/graphgetSuggestedFollowsByActor.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.getSuggestedFollowsByActor + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphGetSuggestedFollowsByActor_Output is the output of a app.bsky.graph.getSuggestedFollowsByActor call. +type GraphGetSuggestedFollowsByActor_Output struct { + // isFallback: If true, response has fallen-back to generic results, and is not scoped using relativeToDid + IsFallback *bool `json:"isFallback,omitempty" cborgen:"isFallback,omitempty"` + // recId: Snowflake for this recommendation, use when submitting recommendation events. + RecId *int64 `json:"recId,omitempty" cborgen:"recId,omitempty"` + Suggestions []*ActorDefs_ProfileView `json:"suggestions" cborgen:"suggestions"` +} + +// GraphGetSuggestedFollowsByActor calls the XRPC method "app.bsky.graph.getSuggestedFollowsByActor". +func GraphGetSuggestedFollowsByActor(ctx context.Context, c lexutil.LexClient, actor string) (*GraphGetSuggestedFollowsByActor_Output, error) { + var out GraphGetSuggestedFollowsByActor_Output + + params := map[string]interface{}{} + params["actor"] = actor + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.getSuggestedFollowsByActor", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphlist.go b/api/bsky/graphlist.go new file mode 100644 index 000000000..d5dd1ea23 --- /dev/null +++ b/api/bsky/graphlist.go @@ -0,0 +1,87 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.list + +package bsky + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func init() { + lexutil.RegisterType("app.bsky.graph.list", &GraphList{}) +} + +type GraphList struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.list"` + Avatar *lexutil.LexBlob `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + DescriptionFacets []*RichtextFacet `json:"descriptionFacets,omitempty" cborgen:"descriptionFacets,omitempty"` + Labels *GraphList_Labels `json:"labels,omitempty" cborgen:"labels,omitempty"` + // name: Display name for list; can not be empty. + Name string `json:"name" cborgen:"name"` + // purpose: Defines the purpose of the list (aka, moderation-oriented or curration-oriented) + Purpose *string `json:"purpose" cborgen:"purpose"` +} + +type GraphList_Labels struct { + LabelDefs_SelfLabels *comatproto.LabelDefs_SelfLabels +} + +func (t *GraphList_Labels) MarshalJSON() ([]byte, error) { + if t.LabelDefs_SelfLabels != nil { + t.LabelDefs_SelfLabels.LexiconTypeID = "com.atproto.label.defs#selfLabels" + return json.Marshal(t.LabelDefs_SelfLabels) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *GraphList_Labels) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return json.Unmarshal(b, t.LabelDefs_SelfLabels) + default: + return nil + } +} + +func (t *GraphList_Labels) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.LabelDefs_SelfLabels != nil { + return t.LabelDefs_SelfLabels.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *GraphList_Labels) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return t.LabelDefs_SelfLabels.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } +} diff --git a/api/bsky/graphlistblock.go b/api/bsky/graphlistblock.go new file mode 100644 index 000000000..1971a18f1 --- /dev/null +++ b/api/bsky/graphlistblock.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.listblock + +package bsky + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.graph.listblock", &GraphListblock{}) +} + +type GraphListblock struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.listblock"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // subject: Reference (AT-URI) to the mod list record. + Subject string `json:"subject" cborgen:"subject"` +} diff --git a/api/bsky/graphlistitem.go b/api/bsky/graphlistitem.go new file mode 100644 index 000000000..18e487f55 --- /dev/null +++ b/api/bsky/graphlistitem.go @@ -0,0 +1,22 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.listitem + +package bsky + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.graph.listitem", &GraphListitem{}) +} + +type GraphListitem struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.listitem"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // list: Reference (AT-URI) to the list record (app.bsky.graph.list). + List string `json:"list" cborgen:"list"` + // subject: The account which is included on the list. + Subject string `json:"subject" cborgen:"subject"` +} diff --git a/api/bsky/graphmute.go b/api/bsky/graphmute.go deleted file mode 100644 index b4f8217bb..000000000 --- a/api/bsky/graphmute.go +++ /dev/null @@ -1,25 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.graph.mute - -func init() { -} - -type GraphMute_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - User string `json:"user" cborgen:"user"` -} - -func GraphMute(ctx context.Context, c *xrpc.Client, input *GraphMute_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "app.bsky.graph.mute", nil, input, nil); err != nil { - return err - } - - return nil -} diff --git a/api/bsky/graphmuteActor.go b/api/bsky/graphmuteActor.go new file mode 100644 index 000000000..ef1e6a246 --- /dev/null +++ b/api/bsky/graphmuteActor.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.muteActor + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphMuteActor_Input is the input argument to a app.bsky.graph.muteActor call. +type GraphMuteActor_Input struct { + Actor string `json:"actor" cborgen:"actor"` +} + +// GraphMuteActor calls the XRPC method "app.bsky.graph.muteActor". +func GraphMuteActor(ctx context.Context, c lexutil.LexClient, input *GraphMuteActor_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.graph.muteActor", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/graphmuteActorList.go b/api/bsky/graphmuteActorList.go new file mode 100644 index 000000000..df3e9fd55 --- /dev/null +++ b/api/bsky/graphmuteActorList.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.muteActorList + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphMuteActorList_Input is the input argument to a app.bsky.graph.muteActorList call. +type GraphMuteActorList_Input struct { + List string `json:"list" cborgen:"list"` +} + +// GraphMuteActorList calls the XRPC method "app.bsky.graph.muteActorList". +func GraphMuteActorList(ctx context.Context, c lexutil.LexClient, input *GraphMuteActorList_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.graph.muteActorList", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/graphmuteThread.go b/api/bsky/graphmuteThread.go new file mode 100644 index 000000000..9f841a9cf --- /dev/null +++ b/api/bsky/graphmuteThread.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.muteThread + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphMuteThread_Input is the input argument to a app.bsky.graph.muteThread call. +type GraphMuteThread_Input struct { + Root string `json:"root" cborgen:"root"` +} + +// GraphMuteThread calls the XRPC method "app.bsky.graph.muteThread". +func GraphMuteThread(ctx context.Context, c lexutil.LexClient, input *GraphMuteThread_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.graph.muteThread", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/graphsearchStarterPacks.go b/api/bsky/graphsearchStarterPacks.go new file mode 100644 index 000000000..4a38491e7 --- /dev/null +++ b/api/bsky/graphsearchStarterPacks.go @@ -0,0 +1,38 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.searchStarterPacks + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphSearchStarterPacks_Output is the output of a app.bsky.graph.searchStarterPacks call. +type GraphSearchStarterPacks_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + StarterPacks []*GraphDefs_StarterPackViewBasic `json:"starterPacks" cborgen:"starterPacks"` +} + +// GraphSearchStarterPacks calls the XRPC method "app.bsky.graph.searchStarterPacks". +// +// q: Search query string. Syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. +func GraphSearchStarterPacks(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, q string) (*GraphSearchStarterPacks_Output, error) { + var out GraphSearchStarterPacks_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["q"] = q + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.graph.searchStarterPacks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphstarterpack.go b/api/bsky/graphstarterpack.go new file mode 100644 index 000000000..d31b3f42d --- /dev/null +++ b/api/bsky/graphstarterpack.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.starterpack + +package bsky + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.graph.starterpack", &GraphStarterpack{}) +} + +type GraphStarterpack struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.starterpack"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + DescriptionFacets []*RichtextFacet `json:"descriptionFacets,omitempty" cborgen:"descriptionFacets,omitempty"` + Feeds []*GraphStarterpack_FeedItem `json:"feeds,omitempty" cborgen:"feeds,omitempty"` + // list: Reference (AT-URI) to the list record. + List string `json:"list" cborgen:"list"` + // name: Display name for starter pack; can not be empty. + Name string `json:"name" cborgen:"name"` +} + +// GraphStarterpack_FeedItem is a "feedItem" in the app.bsky.graph.starterpack schema. +type GraphStarterpack_FeedItem struct { + Uri string `json:"uri" cborgen:"uri"` +} diff --git a/api/bsky/graphunmute.go b/api/bsky/graphunmute.go deleted file mode 100644 index 04d7b774c..000000000 --- a/api/bsky/graphunmute.go +++ /dev/null @@ -1,25 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.graph.unmute - -func init() { -} - -type GraphUnmute_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - User string `json:"user" cborgen:"user"` -} - -func GraphUnmute(ctx context.Context, c *xrpc.Client, input *GraphUnmute_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "app.bsky.graph.unmute", nil, input, nil); err != nil { - return err - } - - return nil -} diff --git a/api/bsky/graphunmuteActor.go b/api/bsky/graphunmuteActor.go new file mode 100644 index 000000000..13849844a --- /dev/null +++ b/api/bsky/graphunmuteActor.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.unmuteActor + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphUnmuteActor_Input is the input argument to a app.bsky.graph.unmuteActor call. +type GraphUnmuteActor_Input struct { + Actor string `json:"actor" cborgen:"actor"` +} + +// GraphUnmuteActor calls the XRPC method "app.bsky.graph.unmuteActor". +func GraphUnmuteActor(ctx context.Context, c lexutil.LexClient, input *GraphUnmuteActor_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.graph.unmuteActor", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/graphunmuteActorList.go b/api/bsky/graphunmuteActorList.go new file mode 100644 index 000000000..6fbf14634 --- /dev/null +++ b/api/bsky/graphunmuteActorList.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.unmuteActorList + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphUnmuteActorList_Input is the input argument to a app.bsky.graph.unmuteActorList call. +type GraphUnmuteActorList_Input struct { + List string `json:"list" cborgen:"list"` +} + +// GraphUnmuteActorList calls the XRPC method "app.bsky.graph.unmuteActorList". +func GraphUnmuteActorList(ctx context.Context, c lexutil.LexClient, input *GraphUnmuteActorList_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.graph.unmuteActorList", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/graphunmuteThread.go b/api/bsky/graphunmuteThread.go new file mode 100644 index 000000000..81f6e4bf9 --- /dev/null +++ b/api/bsky/graphunmuteThread.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.unmuteThread + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// GraphUnmuteThread_Input is the input argument to a app.bsky.graph.unmuteThread call. +type GraphUnmuteThread_Input struct { + Root string `json:"root" cborgen:"root"` +} + +// GraphUnmuteThread calls the XRPC method "app.bsky.graph.unmuteThread". +func GraphUnmuteThread(ctx context.Context, c lexutil.LexClient, input *GraphUnmuteThread_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.graph.unmuteThread", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/graphverification.go b/api/bsky/graphverification.go new file mode 100644 index 000000000..144ee2e66 --- /dev/null +++ b/api/bsky/graphverification.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.graph.verification + +package bsky + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.graph.verification", &GraphVerification{}) +} + +type GraphVerification struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.graph.verification"` + // createdAt: Date of when the verification was created. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // displayName: Display name of the subject the verification applies to at the moment of verifying, which might not be the same at the time of viewing. The verification is only valid if the current displayName matches the one at the time of verifying. + DisplayName string `json:"displayName" cborgen:"displayName"` + // handle: Handle of the subject the verification applies to at the moment of verifying, which might not be the same at the time of viewing. The verification is only valid if the current handle matches the one at the time of verifying. + Handle string `json:"handle" cborgen:"handle"` + // subject: DID of the subject the verification applies to. + Subject string `json:"subject" cborgen:"subject"` +} diff --git a/api/bsky/labelerdefs.go b/api/bsky/labelerdefs.go new file mode 100644 index 000000000..b9dbb38d6 --- /dev/null +++ b/api/bsky/labelerdefs.go @@ -0,0 +1,53 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.labeler.defs + +package bsky + +import ( + comatproto "github.com/bluesky-social/indigo/api/atproto" +) + +// LabelerDefs_LabelerPolicies is a "labelerPolicies" in the app.bsky.labeler.defs schema. +type LabelerDefs_LabelerPolicies struct { + // labelValueDefinitions: Label values created by this labeler and scoped exclusively to it. Labels defined here will override global label definitions for this labeler. + LabelValueDefinitions []*comatproto.LabelDefs_LabelValueDefinition `json:"labelValueDefinitions,omitempty" cborgen:"labelValueDefinitions,omitempty"` + // labelValues: The label values which this labeler publishes. May include global or custom labels. + LabelValues []*string `json:"labelValues" cborgen:"labelValues"` +} + +// LabelerDefs_LabelerView is a "labelerView" in the app.bsky.labeler.defs schema. +type LabelerDefs_LabelerView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.labeler.defs#labelerView"` + Cid string `json:"cid" cborgen:"cid"` + Creator *ActorDefs_ProfileView `json:"creator" cborgen:"creator"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + LikeCount *int64 `json:"likeCount,omitempty" cborgen:"likeCount,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + Viewer *LabelerDefs_LabelerViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// LabelerDefs_LabelerViewDetailed is a "labelerViewDetailed" in the app.bsky.labeler.defs schema. +type LabelerDefs_LabelerViewDetailed struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.labeler.defs#labelerViewDetailed"` + Cid string `json:"cid" cborgen:"cid"` + Creator *ActorDefs_ProfileView `json:"creator" cborgen:"creator"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + LikeCount *int64 `json:"likeCount,omitempty" cborgen:"likeCount,omitempty"` + Policies *LabelerDefs_LabelerPolicies `json:"policies" cborgen:"policies"` + // reasonTypes: The set of report reason 'codes' which are in-scope for this service to review and action. These usually align to policy categories. If not defined (distinct from empty array), all reason types are allowed. + ReasonTypes []*string `json:"reasonTypes,omitempty" cborgen:"reasonTypes,omitempty"` + // subjectCollections: Set of record types (collection NSIDs) which can be reported to this service. If not defined (distinct from empty array), default is any record type. + SubjectCollections []string `json:"subjectCollections,omitempty" cborgen:"subjectCollections,omitempty"` + // subjectTypes: The set of subject types (account, record, etc) this service accepts reports on. + SubjectTypes []*string `json:"subjectTypes,omitempty" cborgen:"subjectTypes,omitempty"` + Uri string `json:"uri" cborgen:"uri"` + Viewer *LabelerDefs_LabelerViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// LabelerDefs_LabelerViewerState is a "labelerViewerState" in the app.bsky.labeler.defs schema. +type LabelerDefs_LabelerViewerState struct { + Like *string `json:"like,omitempty" cborgen:"like,omitempty"` +} diff --git a/api/bsky/labelergetServices.go b/api/bsky/labelergetServices.go new file mode 100644 index 000000000..1b3a73af2 --- /dev/null +++ b/api/bsky/labelergetServices.go @@ -0,0 +1,69 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.labeler.getServices + +package bsky + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// LabelerGetServices_Output is the output of a app.bsky.labeler.getServices call. +type LabelerGetServices_Output struct { + Views []*LabelerGetServices_Output_Views_Elem `json:"views" cborgen:"views"` +} + +type LabelerGetServices_Output_Views_Elem struct { + LabelerDefs_LabelerView *LabelerDefs_LabelerView + LabelerDefs_LabelerViewDetailed *LabelerDefs_LabelerViewDetailed +} + +func (t *LabelerGetServices_Output_Views_Elem) MarshalJSON() ([]byte, error) { + if t.LabelerDefs_LabelerView != nil { + t.LabelerDefs_LabelerView.LexiconTypeID = "app.bsky.labeler.defs#labelerView" + return json.Marshal(t.LabelerDefs_LabelerView) + } + if t.LabelerDefs_LabelerViewDetailed != nil { + t.LabelerDefs_LabelerViewDetailed.LexiconTypeID = "app.bsky.labeler.defs#labelerViewDetailed" + return json.Marshal(t.LabelerDefs_LabelerViewDetailed) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *LabelerGetServices_Output_Views_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.labeler.defs#labelerView": + t.LabelerDefs_LabelerView = new(LabelerDefs_LabelerView) + return json.Unmarshal(b, t.LabelerDefs_LabelerView) + case "app.bsky.labeler.defs#labelerViewDetailed": + t.LabelerDefs_LabelerViewDetailed = new(LabelerDefs_LabelerViewDetailed) + return json.Unmarshal(b, t.LabelerDefs_LabelerViewDetailed) + default: + return nil + } +} + +// LabelerGetServices calls the XRPC method "app.bsky.labeler.getServices". +func LabelerGetServices(ctx context.Context, c lexutil.LexClient, detailed bool, dids []string) (*LabelerGetServices_Output, error) { + var out LabelerGetServices_Output + + params := map[string]interface{}{} + if detailed { + params["detailed"] = detailed + } + params["dids"] = dids + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.labeler.getServices", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/labelerservice.go b/api/bsky/labelerservice.go new file mode 100644 index 000000000..93337775c --- /dev/null +++ b/api/bsky/labelerservice.go @@ -0,0 +1,87 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.labeler.service + +package bsky + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func init() { + lexutil.RegisterType("app.bsky.labeler.service", &LabelerService{}) +} + +type LabelerService struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.labeler.service"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Labels *LabelerService_Labels `json:"labels,omitempty" cborgen:"labels,omitempty"` + Policies *LabelerDefs_LabelerPolicies `json:"policies" cborgen:"policies"` + // reasonTypes: The set of report reason 'codes' which are in-scope for this service to review and action. These usually align to policy categories. If not defined (distinct from empty array), all reason types are allowed. + ReasonTypes []*string `json:"reasonTypes,omitempty" cborgen:"reasonTypes,omitempty"` + // subjectCollections: Set of record types (collection NSIDs) which can be reported to this service. If not defined (distinct from empty array), default is any record type. + SubjectCollections []string `json:"subjectCollections,omitempty" cborgen:"subjectCollections,omitempty"` + // subjectTypes: The set of subject types (account, record, etc) this service accepts reports on. + SubjectTypes []*string `json:"subjectTypes,omitempty" cborgen:"subjectTypes,omitempty"` +} + +type LabelerService_Labels struct { + LabelDefs_SelfLabels *comatproto.LabelDefs_SelfLabels +} + +func (t *LabelerService_Labels) MarshalJSON() ([]byte, error) { + if t.LabelDefs_SelfLabels != nil { + t.LabelDefs_SelfLabels.LexiconTypeID = "com.atproto.label.defs#selfLabels" + return json.Marshal(t.LabelDefs_SelfLabels) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *LabelerService_Labels) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return json.Unmarshal(b, t.LabelDefs_SelfLabels) + default: + return nil + } +} + +func (t *LabelerService_Labels) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.LabelDefs_SelfLabels != nil { + return t.LabelDefs_SelfLabels.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *LabelerService_Labels) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "com.atproto.label.defs#selfLabels": + t.LabelDefs_SelfLabels = new(comatproto.LabelDefs_SelfLabels) + return t.LabelDefs_SelfLabels.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } +} diff --git a/api/bsky/notificationdeclaration.go b/api/bsky/notificationdeclaration.go new file mode 100644 index 000000000..f7fcc8da5 --- /dev/null +++ b/api/bsky/notificationdeclaration.go @@ -0,0 +1,19 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.declaration + +package bsky + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("app.bsky.notification.declaration", &NotificationDeclaration{}) +} + +type NotificationDeclaration struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.notification.declaration"` + // allowSubscriptions: A declaration of the user's preference for allowing activity subscriptions from other users. Absence of a record implies 'followers'. + AllowSubscriptions string `json:"allowSubscriptions" cborgen:"allowSubscriptions"` +} diff --git a/api/bsky/notificationdefs.go b/api/bsky/notificationdefs.go new file mode 100644 index 000000000..8debd58e6 --- /dev/null +++ b/api/bsky/notificationdefs.go @@ -0,0 +1,59 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.defs + +package bsky + +// NotificationDefs_ActivitySubscription is a "activitySubscription" in the app.bsky.notification.defs schema. +type NotificationDefs_ActivitySubscription struct { + Post bool `json:"post" cborgen:"post"` + Reply bool `json:"reply" cborgen:"reply"` +} + +// NotificationDefs_ChatPreference is a "chatPreference" in the app.bsky.notification.defs schema. +type NotificationDefs_ChatPreference struct { + Include string `json:"include" cborgen:"include"` + Push bool `json:"push" cborgen:"push"` +} + +// NotificationDefs_FilterablePreference is a "filterablePreference" in the app.bsky.notification.defs schema. +type NotificationDefs_FilterablePreference struct { + Include string `json:"include" cborgen:"include"` + List bool `json:"list" cborgen:"list"` + Push bool `json:"push" cborgen:"push"` +} + +// NotificationDefs_Preference is a "preference" in the app.bsky.notification.defs schema. +type NotificationDefs_Preference struct { + List bool `json:"list" cborgen:"list"` + Push bool `json:"push" cborgen:"push"` +} + +// NotificationDefs_Preferences is a "preferences" in the app.bsky.notification.defs schema. +type NotificationDefs_Preferences struct { + Chat *NotificationDefs_ChatPreference `json:"chat" cborgen:"chat"` + Follow *NotificationDefs_FilterablePreference `json:"follow" cborgen:"follow"` + Like *NotificationDefs_FilterablePreference `json:"like" cborgen:"like"` + LikeViaRepost *NotificationDefs_FilterablePreference `json:"likeViaRepost" cborgen:"likeViaRepost"` + Mention *NotificationDefs_FilterablePreference `json:"mention" cborgen:"mention"` + Quote *NotificationDefs_FilterablePreference `json:"quote" cborgen:"quote"` + Reply *NotificationDefs_FilterablePreference `json:"reply" cborgen:"reply"` + Repost *NotificationDefs_FilterablePreference `json:"repost" cborgen:"repost"` + RepostViaRepost *NotificationDefs_FilterablePreference `json:"repostViaRepost" cborgen:"repostViaRepost"` + StarterpackJoined *NotificationDefs_Preference `json:"starterpackJoined" cborgen:"starterpackJoined"` + SubscribedPost *NotificationDefs_Preference `json:"subscribedPost" cborgen:"subscribedPost"` + Unverified *NotificationDefs_Preference `json:"unverified" cborgen:"unverified"` + Verified *NotificationDefs_Preference `json:"verified" cborgen:"verified"` +} + +// NotificationDefs_RecordDeleted is a "recordDeleted" in the app.bsky.notification.defs schema. +type NotificationDefs_RecordDeleted struct { +} + +// NotificationDefs_SubjectActivitySubscription is a "subjectActivitySubscription" in the app.bsky.notification.defs schema. +// +// Object used to store activity subscription data in stash. +type NotificationDefs_SubjectActivitySubscription struct { + ActivitySubscription *NotificationDefs_ActivitySubscription `json:"activitySubscription" cborgen:"activitySubscription"` + Subject string `json:"subject" cborgen:"subject"` +} diff --git a/api/bsky/notificationgetCount.go b/api/bsky/notificationgetCount.go deleted file mode 100644 index 9ec5ddc79..000000000 --- a/api/bsky/notificationgetCount.go +++ /dev/null @@ -1,26 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.notification.getCount - -func init() { -} - -type NotificationGetCount_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Count int64 `json:"count" cborgen:"count"` -} - -func NotificationGetCount(ctx context.Context, c *xrpc.Client) (*NotificationGetCount_Output, error) { - var out NotificationGetCount_Output - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.notification.getCount", nil, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/notificationgetPreferences.go b/api/bsky/notificationgetPreferences.go new file mode 100644 index 000000000..3a325fd45 --- /dev/null +++ b/api/bsky/notificationgetPreferences.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.getPreferences + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationGetPreferences_Output is the output of a app.bsky.notification.getPreferences call. +type NotificationGetPreferences_Output struct { + Preferences *NotificationDefs_Preferences `json:"preferences" cborgen:"preferences"` +} + +// NotificationGetPreferences calls the XRPC method "app.bsky.notification.getPreferences". +func NotificationGetPreferences(ctx context.Context, c lexutil.LexClient) (*NotificationGetPreferences_Output, error) { + var out NotificationGetPreferences_Output + + params := map[string]interface{}{} + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.notification.getPreferences", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/notificationgetUnreadCount.go b/api/bsky/notificationgetUnreadCount.go new file mode 100644 index 000000000..4bd7cbf79 --- /dev/null +++ b/api/bsky/notificationgetUnreadCount.go @@ -0,0 +1,34 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.getUnreadCount + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationGetUnreadCount_Output is the output of a app.bsky.notification.getUnreadCount call. +type NotificationGetUnreadCount_Output struct { + Count int64 `json:"count" cborgen:"count"` +} + +// NotificationGetUnreadCount calls the XRPC method "app.bsky.notification.getUnreadCount". +func NotificationGetUnreadCount(ctx context.Context, c lexutil.LexClient, priority bool, seenAt string) (*NotificationGetUnreadCount_Output, error) { + var out NotificationGetUnreadCount_Output + + params := map[string]interface{}{} + if priority { + params["priority"] = priority + } + if seenAt != "" { + params["seenAt"] = seenAt + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.notification.getUnreadCount", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/notificationlist.go b/api/bsky/notificationlist.go deleted file mode 100644 index b4e885e1e..000000000 --- a/api/bsky/notificationlist.go +++ /dev/null @@ -1,45 +0,0 @@ -package schemagen - -import ( - "context" - - "github.com/bluesky-social/indigo/lex/util" - "github.com/bluesky-social/indigo/xrpc" -) - -// schema: app.bsky.notification.list - -func init() { -} - -type NotificationList_Notification struct { - LexiconTypeID string `json:"$type,omitempty"` - Author *ActorRef_WithInfo `json:"author" cborgen:"author"` - Cid string `json:"cid" cborgen:"cid"` - IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` - IsRead bool `json:"isRead" cborgen:"isRead"` - Reason string `json:"reason" cborgen:"reason"` - ReasonSubject *string `json:"reasonSubject,omitempty" cborgen:"reasonSubject"` - Record util.LexiconTypeDecoder `json:"record" cborgen:"record"` - Uri string `json:"uri" cborgen:"uri"` -} - -type NotificationList_Output struct { - LexiconTypeID string `json:"$type,omitempty"` - Cursor *string `json:"cursor,omitempty" cborgen:"cursor"` - Notifications []*NotificationList_Notification `json:"notifications" cborgen:"notifications"` -} - -func NotificationList(ctx context.Context, c *xrpc.Client, before string, limit int64) (*NotificationList_Output, error) { - var out NotificationList_Output - - params := map[string]interface{}{ - "before": before, - "limit": limit, - } - if err := c.Do(ctx, xrpc.Query, "", "app.bsky.notification.list", params, nil, &out); err != nil { - return nil, err - } - - return &out, nil -} diff --git a/api/bsky/notificationlistActivitySubscriptions.go b/api/bsky/notificationlistActivitySubscriptions.go new file mode 100644 index 000000000..0c6451a0b --- /dev/null +++ b/api/bsky/notificationlistActivitySubscriptions.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.listActivitySubscriptions + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationListActivitySubscriptions_Output is the output of a app.bsky.notification.listActivitySubscriptions call. +type NotificationListActivitySubscriptions_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Subscriptions []*ActorDefs_ProfileView `json:"subscriptions" cborgen:"subscriptions"` +} + +// NotificationListActivitySubscriptions calls the XRPC method "app.bsky.notification.listActivitySubscriptions". +func NotificationListActivitySubscriptions(ctx context.Context, c lexutil.LexClient, cursor string, limit int64) (*NotificationListActivitySubscriptions_Output, error) { + var out NotificationListActivitySubscriptions_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.notification.listActivitySubscriptions", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/notificationlistNotifications.go b/api/bsky/notificationlistNotifications.go new file mode 100644 index 000000000..1310c187d --- /dev/null +++ b/api/bsky/notificationlistNotifications.go @@ -0,0 +1,63 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.listNotifications + +package bsky + +import ( + "context" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationListNotifications_Notification is a "notification" in the app.bsky.notification.listNotifications schema. +type NotificationListNotifications_Notification struct { + Author *ActorDefs_ProfileView `json:"author" cborgen:"author"` + Cid string `json:"cid" cborgen:"cid"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + IsRead bool `json:"isRead" cborgen:"isRead"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + // reason: The reason why this notification was delivered - e.g. your post was liked, or you received a new follower. + Reason string `json:"reason" cborgen:"reason"` + ReasonSubject *string `json:"reasonSubject,omitempty" cborgen:"reasonSubject,omitempty"` + Record *lexutil.LexiconTypeDecoder `json:"record" cborgen:"record"` + Uri string `json:"uri" cborgen:"uri"` +} + +// NotificationListNotifications_Output is the output of a app.bsky.notification.listNotifications call. +type NotificationListNotifications_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Notifications []*NotificationListNotifications_Notification `json:"notifications" cborgen:"notifications"` + Priority *bool `json:"priority,omitempty" cborgen:"priority,omitempty"` + SeenAt *string `json:"seenAt,omitempty" cborgen:"seenAt,omitempty"` +} + +// NotificationListNotifications calls the XRPC method "app.bsky.notification.listNotifications". +// +// reasons: Notification reasons to include in response. +func NotificationListNotifications(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, priority bool, reasons []string, seenAt string) (*NotificationListNotifications_Output, error) { + var out NotificationListNotifications_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if priority { + params["priority"] = priority + } + if len(reasons) != 0 { + params["reasons"] = reasons + } + if seenAt != "" { + params["seenAt"] = seenAt + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.notification.listNotifications", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/notificationputActivitySubscription.go b/api/bsky/notificationputActivitySubscription.go new file mode 100644 index 000000000..49619c0dc --- /dev/null +++ b/api/bsky/notificationputActivitySubscription.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.putActivitySubscription + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationPutActivitySubscription_Input is the input argument to a app.bsky.notification.putActivitySubscription call. +type NotificationPutActivitySubscription_Input struct { + ActivitySubscription *NotificationDefs_ActivitySubscription `json:"activitySubscription" cborgen:"activitySubscription"` + Subject string `json:"subject" cborgen:"subject"` +} + +// NotificationPutActivitySubscription_Output is the output of a app.bsky.notification.putActivitySubscription call. +type NotificationPutActivitySubscription_Output struct { + ActivitySubscription *NotificationDefs_ActivitySubscription `json:"activitySubscription,omitempty" cborgen:"activitySubscription,omitempty"` + Subject string `json:"subject" cborgen:"subject"` +} + +// NotificationPutActivitySubscription calls the XRPC method "app.bsky.notification.putActivitySubscription". +func NotificationPutActivitySubscription(ctx context.Context, c lexutil.LexClient, input *NotificationPutActivitySubscription_Input) (*NotificationPutActivitySubscription_Output, error) { + var out NotificationPutActivitySubscription_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.notification.putActivitySubscription", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/notificationputPreferences.go b/api/bsky/notificationputPreferences.go new file mode 100644 index 000000000..bf8f56404 --- /dev/null +++ b/api/bsky/notificationputPreferences.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.putPreferences + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationPutPreferences_Input is the input argument to a app.bsky.notification.putPreferences call. +type NotificationPutPreferences_Input struct { + Priority bool `json:"priority" cborgen:"priority"` +} + +// NotificationPutPreferences calls the XRPC method "app.bsky.notification.putPreferences". +func NotificationPutPreferences(ctx context.Context, c lexutil.LexClient, input *NotificationPutPreferences_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.notification.putPreferences", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/notificationputPreferencesV2.go b/api/bsky/notificationputPreferencesV2.go new file mode 100644 index 000000000..19a65eb91 --- /dev/null +++ b/api/bsky/notificationputPreferencesV2.go @@ -0,0 +1,43 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.putPreferencesV2 + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationPutPreferencesV2_Input is the input argument to a app.bsky.notification.putPreferencesV2 call. +type NotificationPutPreferencesV2_Input struct { + Chat *NotificationDefs_ChatPreference `json:"chat,omitempty" cborgen:"chat,omitempty"` + Follow *NotificationDefs_FilterablePreference `json:"follow,omitempty" cborgen:"follow,omitempty"` + Like *NotificationDefs_FilterablePreference `json:"like,omitempty" cborgen:"like,omitempty"` + LikeViaRepost *NotificationDefs_FilterablePreference `json:"likeViaRepost,omitempty" cborgen:"likeViaRepost,omitempty"` + Mention *NotificationDefs_FilterablePreference `json:"mention,omitempty" cborgen:"mention,omitempty"` + Quote *NotificationDefs_FilterablePreference `json:"quote,omitempty" cborgen:"quote,omitempty"` + Reply *NotificationDefs_FilterablePreference `json:"reply,omitempty" cborgen:"reply,omitempty"` + Repost *NotificationDefs_FilterablePreference `json:"repost,omitempty" cborgen:"repost,omitempty"` + RepostViaRepost *NotificationDefs_FilterablePreference `json:"repostViaRepost,omitempty" cborgen:"repostViaRepost,omitempty"` + StarterpackJoined *NotificationDefs_Preference `json:"starterpackJoined,omitempty" cborgen:"starterpackJoined,omitempty"` + SubscribedPost *NotificationDefs_Preference `json:"subscribedPost,omitempty" cborgen:"subscribedPost,omitempty"` + Unverified *NotificationDefs_Preference `json:"unverified,omitempty" cborgen:"unverified,omitempty"` + Verified *NotificationDefs_Preference `json:"verified,omitempty" cborgen:"verified,omitempty"` +} + +// NotificationPutPreferencesV2_Output is the output of a app.bsky.notification.putPreferencesV2 call. +type NotificationPutPreferencesV2_Output struct { + Preferences *NotificationDefs_Preferences `json:"preferences" cborgen:"preferences"` +} + +// NotificationPutPreferencesV2 calls the XRPC method "app.bsky.notification.putPreferencesV2". +func NotificationPutPreferencesV2(ctx context.Context, c lexutil.LexClient, input *NotificationPutPreferencesV2_Input) (*NotificationPutPreferencesV2_Output, error) { + var out NotificationPutPreferencesV2_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.notification.putPreferencesV2", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/notificationregisterPush.go b/api/bsky/notificationregisterPush.go new file mode 100644 index 000000000..4de00e05b --- /dev/null +++ b/api/bsky/notificationregisterPush.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.registerPush + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationRegisterPush_Input is the input argument to a app.bsky.notification.registerPush call. +type NotificationRegisterPush_Input struct { + // ageRestricted: Set to true when the actor is age restricted + AgeRestricted *bool `json:"ageRestricted,omitempty" cborgen:"ageRestricted,omitempty"` + AppId string `json:"appId" cborgen:"appId"` + Platform string `json:"platform" cborgen:"platform"` + ServiceDid string `json:"serviceDid" cborgen:"serviceDid"` + Token string `json:"token" cborgen:"token"` +} + +// NotificationRegisterPush calls the XRPC method "app.bsky.notification.registerPush". +func NotificationRegisterPush(ctx context.Context, c lexutil.LexClient, input *NotificationRegisterPush_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.notification.registerPush", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/notificationunregisterPush.go b/api/bsky/notificationunregisterPush.go new file mode 100644 index 000000000..f1e8e0f4f --- /dev/null +++ b/api/bsky/notificationunregisterPush.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.unregisterPush + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// NotificationUnregisterPush_Input is the input argument to a app.bsky.notification.unregisterPush call. +type NotificationUnregisterPush_Input struct { + AppId string `json:"appId" cborgen:"appId"` + Platform string `json:"platform" cborgen:"platform"` + ServiceDid string `json:"serviceDid" cborgen:"serviceDid"` + Token string `json:"token" cborgen:"token"` +} + +// NotificationUnregisterPush calls the XRPC method "app.bsky.notification.unregisterPush". +func NotificationUnregisterPush(ctx context.Context, c lexutil.LexClient, input *NotificationUnregisterPush_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.notification.unregisterPush", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/bsky/notificationupdateSeen.go b/api/bsky/notificationupdateSeen.go index e759d3f47..c29410e44 100644 --- a/api/bsky/notificationupdateSeen.go +++ b/api/bsky/notificationupdateSeen.go @@ -1,23 +1,23 @@ -package schemagen +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.notification.updateSeen + +package bsky import ( "context" - "github.com/bluesky-social/indigo/xrpc" + lexutil "github.com/bluesky-social/indigo/lex/util" ) -// schema: app.bsky.notification.updateSeen - -func init() { -} - +// NotificationUpdateSeen_Input is the input argument to a app.bsky.notification.updateSeen call. type NotificationUpdateSeen_Input struct { - LexiconTypeID string `json:"$type,omitempty"` - SeenAt string `json:"seenAt" cborgen:"seenAt"` + SeenAt string `json:"seenAt" cborgen:"seenAt"` } -func NotificationUpdateSeen(ctx context.Context, c *xrpc.Client, input *NotificationUpdateSeen_Input) error { - if err := c.Do(ctx, xrpc.Procedure, "application/json", "app.bsky.notification.updateSeen", nil, input, nil); err != nil { +// NotificationUpdateSeen calls the XRPC method "app.bsky.notification.updateSeen". +func NotificationUpdateSeen(ctx context.Context, c lexutil.LexClient, input *NotificationUpdateSeen_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.notification.updateSeen", nil, input, nil); err != nil { return err } diff --git a/api/bsky/richtextfacet.go b/api/bsky/richtextfacet.go new file mode 100644 index 000000000..a1c808e33 --- /dev/null +++ b/api/bsky/richtextfacet.go @@ -0,0 +1,137 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.richtext.facet + +package bsky + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +// RichtextFacet is a "main" in the app.bsky.richtext.facet schema. +// +// Annotation of a sub-string within rich text. +type RichtextFacet struct { + Features []*RichtextFacet_Features_Elem `json:"features" cborgen:"features"` + Index *RichtextFacet_ByteSlice `json:"index" cborgen:"index"` +} + +// RichtextFacet_ByteSlice is a "byteSlice" in the app.bsky.richtext.facet schema. +// +// Specifies the sub-string range a facet feature applies to. Start index is inclusive, end index is exclusive. Indices are zero-indexed, counting bytes of the UTF-8 encoded text. NOTE: some languages, like Javascript, use UTF-16 or Unicode codepoints for string slice indexing; in these languages, convert to byte arrays before working with facets. +type RichtextFacet_ByteSlice struct { + ByteEnd int64 `json:"byteEnd" cborgen:"byteEnd"` + ByteStart int64 `json:"byteStart" cborgen:"byteStart"` +} + +type RichtextFacet_Features_Elem struct { + RichtextFacet_Mention *RichtextFacet_Mention + RichtextFacet_Link *RichtextFacet_Link + RichtextFacet_Tag *RichtextFacet_Tag +} + +func (t *RichtextFacet_Features_Elem) MarshalJSON() ([]byte, error) { + if t.RichtextFacet_Mention != nil { + t.RichtextFacet_Mention.LexiconTypeID = "app.bsky.richtext.facet#mention" + return json.Marshal(t.RichtextFacet_Mention) + } + if t.RichtextFacet_Link != nil { + t.RichtextFacet_Link.LexiconTypeID = "app.bsky.richtext.facet#link" + return json.Marshal(t.RichtextFacet_Link) + } + if t.RichtextFacet_Tag != nil { + t.RichtextFacet_Tag.LexiconTypeID = "app.bsky.richtext.facet#tag" + return json.Marshal(t.RichtextFacet_Tag) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *RichtextFacet_Features_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.richtext.facet#mention": + t.RichtextFacet_Mention = new(RichtextFacet_Mention) + return json.Unmarshal(b, t.RichtextFacet_Mention) + case "app.bsky.richtext.facet#link": + t.RichtextFacet_Link = new(RichtextFacet_Link) + return json.Unmarshal(b, t.RichtextFacet_Link) + case "app.bsky.richtext.facet#tag": + t.RichtextFacet_Tag = new(RichtextFacet_Tag) + return json.Unmarshal(b, t.RichtextFacet_Tag) + default: + return nil + } +} + +func (t *RichtextFacet_Features_Elem) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.RichtextFacet_Mention != nil { + return t.RichtextFacet_Mention.MarshalCBOR(w) + } + if t.RichtextFacet_Link != nil { + return t.RichtextFacet_Link.MarshalCBOR(w) + } + if t.RichtextFacet_Tag != nil { + return t.RichtextFacet_Tag.MarshalCBOR(w) + } + return fmt.Errorf("can not marshal empty union as CBOR") +} + +func (t *RichtextFacet_Features_Elem) UnmarshalCBOR(r io.Reader) error { + typ, b, err := lexutil.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "app.bsky.richtext.facet#mention": + t.RichtextFacet_Mention = new(RichtextFacet_Mention) + return t.RichtextFacet_Mention.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.richtext.facet#link": + t.RichtextFacet_Link = new(RichtextFacet_Link) + return t.RichtextFacet_Link.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.richtext.facet#tag": + t.RichtextFacet_Tag = new(RichtextFacet_Tag) + return t.RichtextFacet_Tag.UnmarshalCBOR(bytes.NewReader(b)) + default: + return nil + } +} + +// RichtextFacet_Link is a "link" in the app.bsky.richtext.facet schema. +// +// Facet feature for a URL. The text URL may have been simplified or truncated, but the facet reference should be a complete URL. +type RichtextFacet_Link struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.richtext.facet#link"` + Uri string `json:"uri" cborgen:"uri"` +} + +// RichtextFacet_Mention is a "mention" in the app.bsky.richtext.facet schema. +// +// Facet feature for mention of another account. The text is usually a handle, including a '@' prefix, but the facet reference is a DID. +type RichtextFacet_Mention struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.richtext.facet#mention"` + Did string `json:"did" cborgen:"did"` +} + +// RichtextFacet_Tag is a "tag" in the app.bsky.richtext.facet schema. +// +// Facet feature for a hashtag. The text usually includes a '#' prefix, but the facet reference should not (except in the case of 'double hash tags'). +type RichtextFacet_Tag struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.richtext.facet#tag"` + Tag string `json:"tag" cborgen:"tag"` +} diff --git a/api/bsky/systemactorScene.go b/api/bsky/systemactorScene.go deleted file mode 100644 index 6f229a552..000000000 --- a/api/bsky/systemactorScene.go +++ /dev/null @@ -1,8 +0,0 @@ -package schemagen - -// schema: app.bsky.system.actorScene - -func init() { -} - -const SystemActorScene = "app.bsky.system.actorScene" diff --git a/api/bsky/systemactorUser.go b/api/bsky/systemactorUser.go deleted file mode 100644 index 99510fdd2..000000000 --- a/api/bsky/systemactorUser.go +++ /dev/null @@ -1,8 +0,0 @@ -package schemagen - -// schema: app.bsky.system.actorUser - -func init() { -} - -const SystemActorUser = "app.bsky.system.actorUser" diff --git a/api/bsky/systemdeclRef.go b/api/bsky/systemdeclRef.go deleted file mode 100644 index a196206d2..000000000 --- a/api/bsky/systemdeclRef.go +++ /dev/null @@ -1,12 +0,0 @@ -package schemagen - -// schema: app.bsky.system.declRef - -func init() { -} - -type SystemDeclRef struct { - LexiconTypeID string `json:"$type,omitempty"` - ActorType string `json:"actorType" cborgen:"actorType"` - Cid string `json:"cid" cborgen:"cid"` -} diff --git a/api/bsky/systemdeclaration.go b/api/bsky/systemdeclaration.go deleted file mode 100644 index 58d1c2f67..000000000 --- a/api/bsky/systemdeclaration.go +++ /dev/null @@ -1,17 +0,0 @@ -package schemagen - -import ( - "github.com/bluesky-social/indigo/lex/util" -) - -// schema: app.bsky.system.declaration - -func init() { - util.RegisterType("app.bsky.system.declaration", &SystemDeclaration{}) -} - -// RECORDTYPE: SystemDeclaration -type SystemDeclaration struct { - LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.system.declaration"` - ActorType string `json:"actorType" cborgen:"actorType"` -} diff --git a/api/bsky/unspecceddefs.go b/api/bsky/unspecceddefs.go new file mode 100644 index 000000000..2d87c7890 --- /dev/null +++ b/api/bsky/unspecceddefs.go @@ -0,0 +1,116 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.defs + +package bsky + +// UnspeccedDefs_AgeAssuranceEvent is a "ageAssuranceEvent" in the app.bsky.unspecced.defs schema. +// +// Object used to store age assurance data in stash. +type UnspeccedDefs_AgeAssuranceEvent struct { + // attemptId: The unique identifier for this instance of the age assurance flow, in UUID format. + AttemptId string `json:"attemptId" cborgen:"attemptId"` + // completeIp: The IP address used when completing the AA flow. + CompleteIp *string `json:"completeIp,omitempty" cborgen:"completeIp,omitempty"` + // completeUa: The user agent used when completing the AA flow. + CompleteUa *string `json:"completeUa,omitempty" cborgen:"completeUa,omitempty"` + // createdAt: The date and time of this write operation. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // email: The email used for AA. + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + // initIp: The IP address used when initiating the AA flow. + InitIp *string `json:"initIp,omitempty" cborgen:"initIp,omitempty"` + // initUa: The user agent used when initiating the AA flow. + InitUa *string `json:"initUa,omitempty" cborgen:"initUa,omitempty"` + // status: The status of the age assurance process. + Status string `json:"status" cborgen:"status"` +} + +// UnspeccedDefs_AgeAssuranceState is a "ageAssuranceState" in the app.bsky.unspecced.defs schema. +// +// The computed state of the age assurance process, returned to the user in question on certain authenticated requests. +type UnspeccedDefs_AgeAssuranceState struct { + // lastInitiatedAt: The timestamp when this state was last updated. + LastInitiatedAt *string `json:"lastInitiatedAt,omitempty" cborgen:"lastInitiatedAt,omitempty"` + // status: The status of the age assurance process. + Status string `json:"status" cborgen:"status"` +} + +// UnspeccedDefs_SkeletonSearchActor is a "skeletonSearchActor" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_SkeletonSearchActor struct { + Did string `json:"did" cborgen:"did"` +} + +// UnspeccedDefs_SkeletonSearchPost is a "skeletonSearchPost" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_SkeletonSearchPost struct { + Uri string `json:"uri" cborgen:"uri"` +} + +// UnspeccedDefs_SkeletonSearchStarterPack is a "skeletonSearchStarterPack" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_SkeletonSearchStarterPack struct { + Uri string `json:"uri" cborgen:"uri"` +} + +// UnspeccedDefs_SkeletonTrend is a "skeletonTrend" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_SkeletonTrend struct { + Category *string `json:"category,omitempty" cborgen:"category,omitempty"` + Dids []string `json:"dids" cborgen:"dids"` + DisplayName string `json:"displayName" cborgen:"displayName"` + Link string `json:"link" cborgen:"link"` + PostCount int64 `json:"postCount" cborgen:"postCount"` + StartedAt string `json:"startedAt" cborgen:"startedAt"` + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` + Topic string `json:"topic" cborgen:"topic"` +} + +// UnspeccedDefs_ThreadItemBlocked is a "threadItemBlocked" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_ThreadItemBlocked struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.unspecced.defs#threadItemBlocked"` + Author *FeedDefs_BlockedAuthor `json:"author" cborgen:"author"` +} + +// UnspeccedDefs_ThreadItemNoUnauthenticated is a "threadItemNoUnauthenticated" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_ThreadItemNoUnauthenticated struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.unspecced.defs#threadItemNoUnauthenticated"` +} + +// UnspeccedDefs_ThreadItemNotFound is a "threadItemNotFound" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_ThreadItemNotFound struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.unspecced.defs#threadItemNotFound"` +} + +// UnspeccedDefs_ThreadItemPost is a "threadItemPost" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_ThreadItemPost struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=app.bsky.unspecced.defs#threadItemPost"` + // hiddenByThreadgate: The threadgate created by the author indicates this post as a reply to be hidden for everyone consuming the thread. + HiddenByThreadgate bool `json:"hiddenByThreadgate" cborgen:"hiddenByThreadgate"` + // moreParents: This post has more parents that were not present in the response. This is just a boolean, without the number of parents. + MoreParents bool `json:"moreParents" cborgen:"moreParents"` + // moreReplies: This post has more replies that were not present in the response. This is a numeric value, which is best-effort and might not be accurate. + MoreReplies int64 `json:"moreReplies" cborgen:"moreReplies"` + // mutedByViewer: This is by an account muted by the viewer requesting it. + MutedByViewer bool `json:"mutedByViewer" cborgen:"mutedByViewer"` + // opThread: This post is part of a contiguous thread by the OP from the thread root. Many different OP threads can happen in the same thread. + OpThread bool `json:"opThread" cborgen:"opThread"` + Post *FeedDefs_PostView `json:"post" cborgen:"post"` +} + +// UnspeccedDefs_TrendView is a "trendView" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_TrendView struct { + Actors []*ActorDefs_ProfileViewBasic `json:"actors" cborgen:"actors"` + Category *string `json:"category,omitempty" cborgen:"category,omitempty"` + DisplayName string `json:"displayName" cborgen:"displayName"` + Link string `json:"link" cborgen:"link"` + PostCount int64 `json:"postCount" cborgen:"postCount"` + StartedAt string `json:"startedAt" cborgen:"startedAt"` + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` + Topic string `json:"topic" cborgen:"topic"` +} + +// UnspeccedDefs_TrendingTopic is a "trendingTopic" in the app.bsky.unspecced.defs schema. +type UnspeccedDefs_TrendingTopic struct { + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + DisplayName *string `json:"displayName,omitempty" cborgen:"displayName,omitempty"` + Link string `json:"link" cborgen:"link"` + Topic string `json:"topic" cborgen:"topic"` +} diff --git a/api/bsky/unspeccedgetAgeAssuranceState.go b/api/bsky/unspeccedgetAgeAssuranceState.go new file mode 100644 index 000000000..1f7159e27 --- /dev/null +++ b/api/bsky/unspeccedgetAgeAssuranceState.go @@ -0,0 +1,21 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getAgeAssuranceState + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetAgeAssuranceState calls the XRPC method "app.bsky.unspecced.getAgeAssuranceState". +func UnspeccedGetAgeAssuranceState(ctx context.Context, c lexutil.LexClient) (*UnspeccedDefs_AgeAssuranceState, error) { + var out UnspeccedDefs_AgeAssuranceState + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getAgeAssuranceState", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetConfig.go b/api/bsky/unspeccedgetConfig.go new file mode 100644 index 000000000..2fc6f8dd2 --- /dev/null +++ b/api/bsky/unspeccedgetConfig.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getConfig + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetConfig_LiveNowConfig is a "liveNowConfig" in the app.bsky.unspecced.getConfig schema. +type UnspeccedGetConfig_LiveNowConfig struct { + Did string `json:"did" cborgen:"did"` + Domains []string `json:"domains" cborgen:"domains"` +} + +// UnspeccedGetConfig_Output is the output of a app.bsky.unspecced.getConfig call. +type UnspeccedGetConfig_Output struct { + CheckEmailConfirmed *bool `json:"checkEmailConfirmed,omitempty" cborgen:"checkEmailConfirmed,omitempty"` + LiveNow []*UnspeccedGetConfig_LiveNowConfig `json:"liveNow,omitempty" cborgen:"liveNow,omitempty"` +} + +// UnspeccedGetConfig calls the XRPC method "app.bsky.unspecced.getConfig". +func UnspeccedGetConfig(ctx context.Context, c lexutil.LexClient) (*UnspeccedGetConfig_Output, error) { + var out UnspeccedGetConfig_Output + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getConfig", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetOnboardingSuggestedStarterPacks.go b/api/bsky/unspeccedgetOnboardingSuggestedStarterPacks.go new file mode 100644 index 000000000..372aba2ba --- /dev/null +++ b/api/bsky/unspeccedgetOnboardingSuggestedStarterPacks.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getOnboardingSuggestedStarterPacks + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetOnboardingSuggestedStarterPacks_Output is the output of a app.bsky.unspecced.getOnboardingSuggestedStarterPacks call. +type UnspeccedGetOnboardingSuggestedStarterPacks_Output struct { + StarterPacks []*GraphDefs_StarterPackView `json:"starterPacks" cborgen:"starterPacks"` +} + +// UnspeccedGetOnboardingSuggestedStarterPacks calls the XRPC method "app.bsky.unspecced.getOnboardingSuggestedStarterPacks". +func UnspeccedGetOnboardingSuggestedStarterPacks(ctx context.Context, c lexutil.LexClient, limit int64) (*UnspeccedGetOnboardingSuggestedStarterPacks_Output, error) { + var out UnspeccedGetOnboardingSuggestedStarterPacks_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getOnboardingSuggestedStarterPacks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetOnboardingSuggestedStarterPacksSkeleton.go b/api/bsky/unspeccedgetOnboardingSuggestedStarterPacksSkeleton.go new file mode 100644 index 000000000..7033059c5 --- /dev/null +++ b/api/bsky/unspeccedgetOnboardingSuggestedStarterPacksSkeleton.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getOnboardingSuggestedStarterPacksSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetOnboardingSuggestedStarterPacksSkeleton_Output is the output of a app.bsky.unspecced.getOnboardingSuggestedStarterPacksSkeleton call. +type UnspeccedGetOnboardingSuggestedStarterPacksSkeleton_Output struct { + StarterPacks []string `json:"starterPacks" cborgen:"starterPacks"` +} + +// UnspeccedGetOnboardingSuggestedStarterPacksSkeleton calls the XRPC method "app.bsky.unspecced.getOnboardingSuggestedStarterPacksSkeleton". +// +// viewer: DID of the account making the request (not included for public/unauthenticated queries). +func UnspeccedGetOnboardingSuggestedStarterPacksSkeleton(ctx context.Context, c lexutil.LexClient, limit int64, viewer string) (*UnspeccedGetOnboardingSuggestedStarterPacksSkeleton_Output, error) { + var out UnspeccedGetOnboardingSuggestedStarterPacksSkeleton_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getOnboardingSuggestedStarterPacksSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetPopularFeedGenerators.go b/api/bsky/unspeccedgetPopularFeedGenerators.go new file mode 100644 index 000000000..8feaf827b --- /dev/null +++ b/api/bsky/unspeccedgetPopularFeedGenerators.go @@ -0,0 +1,38 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getPopularFeedGenerators + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetPopularFeedGenerators_Output is the output of a app.bsky.unspecced.getPopularFeedGenerators call. +type UnspeccedGetPopularFeedGenerators_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feeds []*FeedDefs_GeneratorView `json:"feeds" cborgen:"feeds"` +} + +// UnspeccedGetPopularFeedGenerators calls the XRPC method "app.bsky.unspecced.getPopularFeedGenerators". +func UnspeccedGetPopularFeedGenerators(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, query string) (*UnspeccedGetPopularFeedGenerators_Output, error) { + var out UnspeccedGetPopularFeedGenerators_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if query != "" { + params["query"] = query + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getPopularFeedGenerators", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetPostThreadOtherV2.go b/api/bsky/unspeccedgetPostThreadOtherV2.go new file mode 100644 index 000000000..e424a8146 --- /dev/null +++ b/api/bsky/unspeccedgetPostThreadOtherV2.go @@ -0,0 +1,69 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getPostThreadOtherV2 + +package bsky + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetPostThreadOtherV2_Output is the output of a app.bsky.unspecced.getPostThreadOtherV2 call. +type UnspeccedGetPostThreadOtherV2_Output struct { + // thread: A flat list of other thread items. The depth of each item is indicated by the depth property inside the item. + Thread []*UnspeccedGetPostThreadOtherV2_ThreadItem `json:"thread" cborgen:"thread"` +} + +// UnspeccedGetPostThreadOtherV2_ThreadItem is a "threadItem" in the app.bsky.unspecced.getPostThreadOtherV2 schema. +type UnspeccedGetPostThreadOtherV2_ThreadItem struct { + // depth: The nesting level of this item in the thread. Depth 0 means the anchor item. Items above have negative depths, items below have positive depths. + Depth int64 `json:"depth" cborgen:"depth"` + Uri string `json:"uri" cborgen:"uri"` + Value *UnspeccedGetPostThreadOtherV2_ThreadItem_Value `json:"value" cborgen:"value"` +} + +type UnspeccedGetPostThreadOtherV2_ThreadItem_Value struct { + UnspeccedDefs_ThreadItemPost *UnspeccedDefs_ThreadItemPost +} + +func (t *UnspeccedGetPostThreadOtherV2_ThreadItem_Value) MarshalJSON() ([]byte, error) { + if t.UnspeccedDefs_ThreadItemPost != nil { + t.UnspeccedDefs_ThreadItemPost.LexiconTypeID = "app.bsky.unspecced.defs#threadItemPost" + return json.Marshal(t.UnspeccedDefs_ThreadItemPost) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *UnspeccedGetPostThreadOtherV2_ThreadItem_Value) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.unspecced.defs#threadItemPost": + t.UnspeccedDefs_ThreadItemPost = new(UnspeccedDefs_ThreadItemPost) + return json.Unmarshal(b, t.UnspeccedDefs_ThreadItemPost) + default: + return nil + } +} + +// UnspeccedGetPostThreadOtherV2 calls the XRPC method "app.bsky.unspecced.getPostThreadOtherV2". +// +// anchor: Reference (AT-URI) to post record. This is the anchor post. +func UnspeccedGetPostThreadOtherV2(ctx context.Context, c lexutil.LexClient, anchor string) (*UnspeccedGetPostThreadOtherV2_Output, error) { + var out UnspeccedGetPostThreadOtherV2_Output + + params := map[string]interface{}{} + params["anchor"] = anchor + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getPostThreadOtherV2", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetPostThreadV2.go b/api/bsky/unspeccedgetPostThreadV2.go new file mode 100644 index 000000000..344a0fbf3 --- /dev/null +++ b/api/bsky/unspeccedgetPostThreadV2.go @@ -0,0 +1,112 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getPostThreadV2 + +package bsky + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetPostThreadV2_Output is the output of a app.bsky.unspecced.getPostThreadV2 call. +type UnspeccedGetPostThreadV2_Output struct { + // hasOtherReplies: Whether this thread has additional replies. If true, a call can be made to the `getPostThreadOtherV2` endpoint to retrieve them. + HasOtherReplies bool `json:"hasOtherReplies" cborgen:"hasOtherReplies"` + // thread: A flat list of thread items. The depth of each item is indicated by the depth property inside the item. + Thread []*UnspeccedGetPostThreadV2_ThreadItem `json:"thread" cborgen:"thread"` + Threadgate *FeedDefs_ThreadgateView `json:"threadgate,omitempty" cborgen:"threadgate,omitempty"` +} + +// UnspeccedGetPostThreadV2_ThreadItem is a "threadItem" in the app.bsky.unspecced.getPostThreadV2 schema. +type UnspeccedGetPostThreadV2_ThreadItem struct { + // depth: The nesting level of this item in the thread. Depth 0 means the anchor item. Items above have negative depths, items below have positive depths. + Depth int64 `json:"depth" cborgen:"depth"` + Uri string `json:"uri" cborgen:"uri"` + Value *UnspeccedGetPostThreadV2_ThreadItem_Value `json:"value" cborgen:"value"` +} + +type UnspeccedGetPostThreadV2_ThreadItem_Value struct { + UnspeccedDefs_ThreadItemPost *UnspeccedDefs_ThreadItemPost + UnspeccedDefs_ThreadItemNoUnauthenticated *UnspeccedDefs_ThreadItemNoUnauthenticated + UnspeccedDefs_ThreadItemNotFound *UnspeccedDefs_ThreadItemNotFound + UnspeccedDefs_ThreadItemBlocked *UnspeccedDefs_ThreadItemBlocked +} + +func (t *UnspeccedGetPostThreadV2_ThreadItem_Value) MarshalJSON() ([]byte, error) { + if t.UnspeccedDefs_ThreadItemPost != nil { + t.UnspeccedDefs_ThreadItemPost.LexiconTypeID = "app.bsky.unspecced.defs#threadItemPost" + return json.Marshal(t.UnspeccedDefs_ThreadItemPost) + } + if t.UnspeccedDefs_ThreadItemNoUnauthenticated != nil { + t.UnspeccedDefs_ThreadItemNoUnauthenticated.LexiconTypeID = "app.bsky.unspecced.defs#threadItemNoUnauthenticated" + return json.Marshal(t.UnspeccedDefs_ThreadItemNoUnauthenticated) + } + if t.UnspeccedDefs_ThreadItemNotFound != nil { + t.UnspeccedDefs_ThreadItemNotFound.LexiconTypeID = "app.bsky.unspecced.defs#threadItemNotFound" + return json.Marshal(t.UnspeccedDefs_ThreadItemNotFound) + } + if t.UnspeccedDefs_ThreadItemBlocked != nil { + t.UnspeccedDefs_ThreadItemBlocked.LexiconTypeID = "app.bsky.unspecced.defs#threadItemBlocked" + return json.Marshal(t.UnspeccedDefs_ThreadItemBlocked) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *UnspeccedGetPostThreadV2_ThreadItem_Value) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.unspecced.defs#threadItemPost": + t.UnspeccedDefs_ThreadItemPost = new(UnspeccedDefs_ThreadItemPost) + return json.Unmarshal(b, t.UnspeccedDefs_ThreadItemPost) + case "app.bsky.unspecced.defs#threadItemNoUnauthenticated": + t.UnspeccedDefs_ThreadItemNoUnauthenticated = new(UnspeccedDefs_ThreadItemNoUnauthenticated) + return json.Unmarshal(b, t.UnspeccedDefs_ThreadItemNoUnauthenticated) + case "app.bsky.unspecced.defs#threadItemNotFound": + t.UnspeccedDefs_ThreadItemNotFound = new(UnspeccedDefs_ThreadItemNotFound) + return json.Unmarshal(b, t.UnspeccedDefs_ThreadItemNotFound) + case "app.bsky.unspecced.defs#threadItemBlocked": + t.UnspeccedDefs_ThreadItemBlocked = new(UnspeccedDefs_ThreadItemBlocked) + return json.Unmarshal(b, t.UnspeccedDefs_ThreadItemBlocked) + default: + return nil + } +} + +// UnspeccedGetPostThreadV2 calls the XRPC method "app.bsky.unspecced.getPostThreadV2". +// +// above: Whether to include parents above the anchor. +// anchor: Reference (AT-URI) to post record. This is the anchor post, and the thread will be built around it. It can be any post in the tree, not necessarily a root post. +// below: How many levels of replies to include below the anchor. +// branchingFactor: Maximum of replies to include at each level of the thread, except for the direct replies to the anchor, which are (NOTE: currently, during unspecced phase) all returned (NOTE: later they might be paginated). +// sort: Sorting for the thread replies. +func UnspeccedGetPostThreadV2(ctx context.Context, c lexutil.LexClient, above bool, anchor string, below int64, branchingFactor int64, sort string) (*UnspeccedGetPostThreadV2_Output, error) { + var out UnspeccedGetPostThreadV2_Output + + params := map[string]interface{}{} + if above { + params["above"] = above + } + params["anchor"] = anchor + if below != 0 { + params["below"] = below + } + if branchingFactor != 0 { + params["branchingFactor"] = branchingFactor + } + if sort != "" { + params["sort"] = sort + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getPostThreadV2", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetSuggestedFeeds.go b/api/bsky/unspeccedgetSuggestedFeeds.go new file mode 100644 index 000000000..917716047 --- /dev/null +++ b/api/bsky/unspeccedgetSuggestedFeeds.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getSuggestedFeeds + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetSuggestedFeeds_Output is the output of a app.bsky.unspecced.getSuggestedFeeds call. +type UnspeccedGetSuggestedFeeds_Output struct { + Feeds []*FeedDefs_GeneratorView `json:"feeds" cborgen:"feeds"` +} + +// UnspeccedGetSuggestedFeeds calls the XRPC method "app.bsky.unspecced.getSuggestedFeeds". +func UnspeccedGetSuggestedFeeds(ctx context.Context, c lexutil.LexClient, limit int64) (*UnspeccedGetSuggestedFeeds_Output, error) { + var out UnspeccedGetSuggestedFeeds_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getSuggestedFeeds", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetSuggestedFeedsSkeleton.go b/api/bsky/unspeccedgetSuggestedFeedsSkeleton.go new file mode 100644 index 000000000..0aacb2706 --- /dev/null +++ b/api/bsky/unspeccedgetSuggestedFeedsSkeleton.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getSuggestedFeedsSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetSuggestedFeedsSkeleton_Output is the output of a app.bsky.unspecced.getSuggestedFeedsSkeleton call. +type UnspeccedGetSuggestedFeedsSkeleton_Output struct { + Feeds []string `json:"feeds" cborgen:"feeds"` +} + +// UnspeccedGetSuggestedFeedsSkeleton calls the XRPC method "app.bsky.unspecced.getSuggestedFeedsSkeleton". +// +// viewer: DID of the account making the request (not included for public/unauthenticated queries). +func UnspeccedGetSuggestedFeedsSkeleton(ctx context.Context, c lexutil.LexClient, limit int64, viewer string) (*UnspeccedGetSuggestedFeedsSkeleton_Output, error) { + var out UnspeccedGetSuggestedFeedsSkeleton_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getSuggestedFeedsSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetSuggestedStarterPacks.go b/api/bsky/unspeccedgetSuggestedStarterPacks.go new file mode 100644 index 000000000..3f61242f5 --- /dev/null +++ b/api/bsky/unspeccedgetSuggestedStarterPacks.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getSuggestedStarterPacks + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetSuggestedStarterPacks_Output is the output of a app.bsky.unspecced.getSuggestedStarterPacks call. +type UnspeccedGetSuggestedStarterPacks_Output struct { + StarterPacks []*GraphDefs_StarterPackView `json:"starterPacks" cborgen:"starterPacks"` +} + +// UnspeccedGetSuggestedStarterPacks calls the XRPC method "app.bsky.unspecced.getSuggestedStarterPacks". +func UnspeccedGetSuggestedStarterPacks(ctx context.Context, c lexutil.LexClient, limit int64) (*UnspeccedGetSuggestedStarterPacks_Output, error) { + var out UnspeccedGetSuggestedStarterPacks_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getSuggestedStarterPacks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetSuggestedStarterPacksSkeleton.go b/api/bsky/unspeccedgetSuggestedStarterPacksSkeleton.go new file mode 100644 index 000000000..09c2eec16 --- /dev/null +++ b/api/bsky/unspeccedgetSuggestedStarterPacksSkeleton.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getSuggestedStarterPacksSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetSuggestedStarterPacksSkeleton_Output is the output of a app.bsky.unspecced.getSuggestedStarterPacksSkeleton call. +type UnspeccedGetSuggestedStarterPacksSkeleton_Output struct { + StarterPacks []string `json:"starterPacks" cborgen:"starterPacks"` +} + +// UnspeccedGetSuggestedStarterPacksSkeleton calls the XRPC method "app.bsky.unspecced.getSuggestedStarterPacksSkeleton". +// +// viewer: DID of the account making the request (not included for public/unauthenticated queries). +func UnspeccedGetSuggestedStarterPacksSkeleton(ctx context.Context, c lexutil.LexClient, limit int64, viewer string) (*UnspeccedGetSuggestedStarterPacksSkeleton_Output, error) { + var out UnspeccedGetSuggestedStarterPacksSkeleton_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getSuggestedStarterPacksSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetSuggestedUsers.go b/api/bsky/unspeccedgetSuggestedUsers.go new file mode 100644 index 000000000..81dcc420f --- /dev/null +++ b/api/bsky/unspeccedgetSuggestedUsers.go @@ -0,0 +1,38 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getSuggestedUsers + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetSuggestedUsers_Output is the output of a app.bsky.unspecced.getSuggestedUsers call. +type UnspeccedGetSuggestedUsers_Output struct { + Actors []*ActorDefs_ProfileView `json:"actors" cborgen:"actors"` + // recId: Snowflake for this recommendation, use when submitting recommendation events. + RecId *string `json:"recId,omitempty" cborgen:"recId,omitempty"` +} + +// UnspeccedGetSuggestedUsers calls the XRPC method "app.bsky.unspecced.getSuggestedUsers". +// +// category: Category of users to get suggestions for. +func UnspeccedGetSuggestedUsers(ctx context.Context, c lexutil.LexClient, category string, limit int64) (*UnspeccedGetSuggestedUsers_Output, error) { + var out UnspeccedGetSuggestedUsers_Output + + params := map[string]interface{}{} + if category != "" { + params["category"] = category + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getSuggestedUsers", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetSuggestedUsersSkeleton.go b/api/bsky/unspeccedgetSuggestedUsersSkeleton.go new file mode 100644 index 000000000..dfaaca33d --- /dev/null +++ b/api/bsky/unspeccedgetSuggestedUsersSkeleton.go @@ -0,0 +1,42 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getSuggestedUsersSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetSuggestedUsersSkeleton_Output is the output of a app.bsky.unspecced.getSuggestedUsersSkeleton call. +type UnspeccedGetSuggestedUsersSkeleton_Output struct { + Dids []string `json:"dids" cborgen:"dids"` + // recId: Snowflake for this recommendation, use when submitting recommendation events. + RecId *string `json:"recId,omitempty" cborgen:"recId,omitempty"` +} + +// UnspeccedGetSuggestedUsersSkeleton calls the XRPC method "app.bsky.unspecced.getSuggestedUsersSkeleton". +// +// category: Category of users to get suggestions for. +// viewer: DID of the account making the request (not included for public/unauthenticated queries). +func UnspeccedGetSuggestedUsersSkeleton(ctx context.Context, c lexutil.LexClient, category string, limit int64, viewer string) (*UnspeccedGetSuggestedUsersSkeleton_Output, error) { + var out UnspeccedGetSuggestedUsersSkeleton_Output + + params := map[string]interface{}{} + if category != "" { + params["category"] = category + } + if limit != 0 { + params["limit"] = limit + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getSuggestedUsersSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetSuggestionsSkeleton.go b/api/bsky/unspeccedgetSuggestionsSkeleton.go new file mode 100644 index 000000000..1ac1973fc --- /dev/null +++ b/api/bsky/unspeccedgetSuggestionsSkeleton.go @@ -0,0 +1,48 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getSuggestionsSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetSuggestionsSkeleton_Output is the output of a app.bsky.unspecced.getSuggestionsSkeleton call. +type UnspeccedGetSuggestionsSkeleton_Output struct { + Actors []*UnspeccedDefs_SkeletonSearchActor `json:"actors" cborgen:"actors"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // recId: Snowflake for this recommendation, use when submitting recommendation events. + RecId *int64 `json:"recId,omitempty" cborgen:"recId,omitempty"` + // relativeToDid: DID of the account these suggestions are relative to. If this is returned undefined, suggestions are based on the viewer. + RelativeToDid *string `json:"relativeToDid,omitempty" cborgen:"relativeToDid,omitempty"` +} + +// UnspeccedGetSuggestionsSkeleton calls the XRPC method "app.bsky.unspecced.getSuggestionsSkeleton". +// +// relativeToDid: DID of the account to get suggestions relative to. If not provided, suggestions will be based on the viewer. +// viewer: DID of the account making the request (not included for public/unauthenticated queries). Used to boost followed accounts in ranking. +func UnspeccedGetSuggestionsSkeleton(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, relativeToDid string, viewer string) (*UnspeccedGetSuggestionsSkeleton_Output, error) { + var out UnspeccedGetSuggestionsSkeleton_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if relativeToDid != "" { + params["relativeToDid"] = relativeToDid + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getSuggestionsSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetTaggedSuggestions.go b/api/bsky/unspeccedgetTaggedSuggestions.go new file mode 100644 index 000000000..7db029d2d --- /dev/null +++ b/api/bsky/unspeccedgetTaggedSuggestions.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getTaggedSuggestions + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetTaggedSuggestions_Output is the output of a app.bsky.unspecced.getTaggedSuggestions call. +type UnspeccedGetTaggedSuggestions_Output struct { + Suggestions []*UnspeccedGetTaggedSuggestions_Suggestion `json:"suggestions" cborgen:"suggestions"` +} + +// UnspeccedGetTaggedSuggestions_Suggestion is a "suggestion" in the app.bsky.unspecced.getTaggedSuggestions schema. +type UnspeccedGetTaggedSuggestions_Suggestion struct { + Subject string `json:"subject" cborgen:"subject"` + SubjectType string `json:"subjectType" cborgen:"subjectType"` + Tag string `json:"tag" cborgen:"tag"` +} + +// UnspeccedGetTaggedSuggestions calls the XRPC method "app.bsky.unspecced.getTaggedSuggestions". +func UnspeccedGetTaggedSuggestions(ctx context.Context, c lexutil.LexClient) (*UnspeccedGetTaggedSuggestions_Output, error) { + var out UnspeccedGetTaggedSuggestions_Output + + params := map[string]interface{}{} + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getTaggedSuggestions", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetTrendingTopics.go b/api/bsky/unspeccedgetTrendingTopics.go new file mode 100644 index 000000000..76b156cc4 --- /dev/null +++ b/api/bsky/unspeccedgetTrendingTopics.go @@ -0,0 +1,37 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getTrendingTopics + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetTrendingTopics_Output is the output of a app.bsky.unspecced.getTrendingTopics call. +type UnspeccedGetTrendingTopics_Output struct { + Suggested []*UnspeccedDefs_TrendingTopic `json:"suggested" cborgen:"suggested"` + Topics []*UnspeccedDefs_TrendingTopic `json:"topics" cborgen:"topics"` +} + +// UnspeccedGetTrendingTopics calls the XRPC method "app.bsky.unspecced.getTrendingTopics". +// +// viewer: DID of the account making the request (not included for public/unauthenticated queries). Used to boost followed accounts in ranking. +func UnspeccedGetTrendingTopics(ctx context.Context, c lexutil.LexClient, limit int64, viewer string) (*UnspeccedGetTrendingTopics_Output, error) { + var out UnspeccedGetTrendingTopics_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getTrendingTopics", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetTrends.go b/api/bsky/unspeccedgetTrends.go new file mode 100644 index 000000000..8dfc9b276 --- /dev/null +++ b/api/bsky/unspeccedgetTrends.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getTrends + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetTrends_Output is the output of a app.bsky.unspecced.getTrends call. +type UnspeccedGetTrends_Output struct { + Trends []*UnspeccedDefs_TrendView `json:"trends" cborgen:"trends"` +} + +// UnspeccedGetTrends calls the XRPC method "app.bsky.unspecced.getTrends". +func UnspeccedGetTrends(ctx context.Context, c lexutil.LexClient, limit int64) (*UnspeccedGetTrends_Output, error) { + var out UnspeccedGetTrends_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getTrends", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedgetTrendsSkeleton.go b/api/bsky/unspeccedgetTrendsSkeleton.go new file mode 100644 index 000000000..5817eaa84 --- /dev/null +++ b/api/bsky/unspeccedgetTrendsSkeleton.go @@ -0,0 +1,36 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.getTrendsSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedGetTrendsSkeleton_Output is the output of a app.bsky.unspecced.getTrendsSkeleton call. +type UnspeccedGetTrendsSkeleton_Output struct { + Trends []*UnspeccedDefs_SkeletonTrend `json:"trends" cborgen:"trends"` +} + +// UnspeccedGetTrendsSkeleton calls the XRPC method "app.bsky.unspecced.getTrendsSkeleton". +// +// viewer: DID of the account making the request (not included for public/unauthenticated queries). +func UnspeccedGetTrendsSkeleton(ctx context.Context, c lexutil.LexClient, limit int64, viewer string) (*UnspeccedGetTrendsSkeleton_Output, error) { + var out UnspeccedGetTrendsSkeleton_Output + + params := map[string]interface{}{} + if limit != 0 { + params["limit"] = limit + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.getTrendsSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedinitAgeAssurance.go b/api/bsky/unspeccedinitAgeAssurance.go new file mode 100644 index 000000000..26edc4a7f --- /dev/null +++ b/api/bsky/unspeccedinitAgeAssurance.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.initAgeAssurance + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedInitAgeAssurance_Input is the input argument to a app.bsky.unspecced.initAgeAssurance call. +type UnspeccedInitAgeAssurance_Input struct { + // countryCode: An ISO 3166-1 alpha-2 code of the user's location. + CountryCode string `json:"countryCode" cborgen:"countryCode"` + // email: The user's email address to receive assurance instructions. + Email string `json:"email" cborgen:"email"` + // language: The user's preferred language for communication during the assurance process. + Language string `json:"language" cborgen:"language"` +} + +// UnspeccedInitAgeAssurance calls the XRPC method "app.bsky.unspecced.initAgeAssurance". +func UnspeccedInitAgeAssurance(ctx context.Context, c lexutil.LexClient, input *UnspeccedInitAgeAssurance_Input) (*UnspeccedDefs_AgeAssuranceState, error) { + var out UnspeccedDefs_AgeAssuranceState + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "app.bsky.unspecced.initAgeAssurance", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedsearchActorsSkeleton.go b/api/bsky/unspeccedsearchActorsSkeleton.go new file mode 100644 index 000000000..ee7ef1114 --- /dev/null +++ b/api/bsky/unspeccedsearchActorsSkeleton.go @@ -0,0 +1,49 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.searchActorsSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedSearchActorsSkeleton_Output is the output of a app.bsky.unspecced.searchActorsSkeleton call. +type UnspeccedSearchActorsSkeleton_Output struct { + Actors []*UnspeccedDefs_SkeletonSearchActor `json:"actors" cborgen:"actors"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // hitsTotal: Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. + HitsTotal *int64 `json:"hitsTotal,omitempty" cborgen:"hitsTotal,omitempty"` +} + +// UnspeccedSearchActorsSkeleton calls the XRPC method "app.bsky.unspecced.searchActorsSkeleton". +// +// cursor: Optional pagination mechanism; may not necessarily allow scrolling through entire result set. +// q: Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax. +// typeahead: If true, acts as fast/simple 'typeahead' query. +// viewer: DID of the account making the request (not included for public/unauthenticated queries). Used to boost followed accounts in ranking. +func UnspeccedSearchActorsSkeleton(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, q string, typeahead bool, viewer string) (*UnspeccedSearchActorsSkeleton_Output, error) { + var out UnspeccedSearchActorsSkeleton_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["q"] = q + if typeahead { + params["typeahead"] = typeahead + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.searchActorsSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedsearchPostsSkeleton.go b/api/bsky/unspeccedsearchPostsSkeleton.go new file mode 100644 index 000000000..3d9bdad73 --- /dev/null +++ b/api/bsky/unspeccedsearchPostsSkeleton.go @@ -0,0 +1,81 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.searchPostsSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedSearchPostsSkeleton_Output is the output of a app.bsky.unspecced.searchPostsSkeleton call. +type UnspeccedSearchPostsSkeleton_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // hitsTotal: Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. + HitsTotal *int64 `json:"hitsTotal,omitempty" cborgen:"hitsTotal,omitempty"` + Posts []*UnspeccedDefs_SkeletonSearchPost `json:"posts" cborgen:"posts"` +} + +// UnspeccedSearchPostsSkeleton calls the XRPC method "app.bsky.unspecced.searchPostsSkeleton". +// +// author: Filter to posts by the given account. Handles are resolved to DID before query-time. +// cursor: Optional pagination mechanism; may not necessarily allow scrolling through entire result set. +// domain: Filter to posts with URLs (facet links or embeds) linking to the given domain (hostname). Server may apply hostname normalization. +// lang: Filter to posts in the given language. Expected to be based on post language field, though server may override language detection. +// mentions: Filter to posts which mention the given account. Handles are resolved to DID before query-time. Only matches rich-text facet mentions. +// q: Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. +// since: Filter results for posts after the indicated datetime (inclusive). Expected to use 'sortAt' timestamp, which may not match 'createdAt'. Can be a datetime, or just an ISO date (YYYY-MM-DD). +// sort: Specifies the ranking order of results. +// tag: Filter to posts with the given tag (hashtag), based on rich-text facet or tag field. Do not include the hash (#) prefix. Multiple tags can be specified, with 'AND' matching. +// until: Filter results for posts before the indicated datetime (not inclusive). Expected to use 'sortAt' timestamp, which may not match 'createdAt'. Can be a datetime, or just an ISO date (YYY-MM-DD). +// url: Filter to posts with links (facet links or embeds) pointing to this URL. Server may apply URL normalization or fuzzy matching. +// viewer: DID of the account making the request (not included for public/unauthenticated queries). Used for 'from:me' queries. +func UnspeccedSearchPostsSkeleton(ctx context.Context, c lexutil.LexClient, author string, cursor string, domain string, lang string, limit int64, mentions string, q string, since string, sort string, tag []string, until string, url string, viewer string) (*UnspeccedSearchPostsSkeleton_Output, error) { + var out UnspeccedSearchPostsSkeleton_Output + + params := map[string]interface{}{} + if author != "" { + params["author"] = author + } + if cursor != "" { + params["cursor"] = cursor + } + if domain != "" { + params["domain"] = domain + } + if lang != "" { + params["lang"] = lang + } + if limit != 0 { + params["limit"] = limit + } + if mentions != "" { + params["mentions"] = mentions + } + params["q"] = q + if since != "" { + params["since"] = since + } + if sort != "" { + params["sort"] = sort + } + if len(tag) != 0 { + params["tag"] = tag + } + if until != "" { + params["until"] = until + } + if url != "" { + params["url"] = url + } + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.searchPostsSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/unspeccedsearchStarterPacksSkeleton.go b/api/bsky/unspeccedsearchStarterPacksSkeleton.go new file mode 100644 index 000000000..0fef0af26 --- /dev/null +++ b/api/bsky/unspeccedsearchStarterPacksSkeleton.go @@ -0,0 +1,45 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.unspecced.searchStarterPacksSkeleton + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// UnspeccedSearchStarterPacksSkeleton_Output is the output of a app.bsky.unspecced.searchStarterPacksSkeleton call. +type UnspeccedSearchStarterPacksSkeleton_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // hitsTotal: Count of search hits. Optional, may be rounded/truncated, and may not be possible to paginate through all hits. + HitsTotal *int64 `json:"hitsTotal,omitempty" cborgen:"hitsTotal,omitempty"` + StarterPacks []*UnspeccedDefs_SkeletonSearchStarterPack `json:"starterPacks" cborgen:"starterPacks"` +} + +// UnspeccedSearchStarterPacksSkeleton calls the XRPC method "app.bsky.unspecced.searchStarterPacksSkeleton". +// +// cursor: Optional pagination mechanism; may not necessarily allow scrolling through entire result set. +// q: Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. +// viewer: DID of the account making the request (not included for public/unauthenticated queries). +func UnspeccedSearchStarterPacksSkeleton(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, q string, viewer string) (*UnspeccedSearchStarterPacksSkeleton_Output, error) { + var out UnspeccedSearchStarterPacksSkeleton_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["q"] = q + if viewer != "" { + params["viewer"] = viewer + } + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.unspecced.searchStarterPacksSkeleton", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/videodefs.go b/api/bsky/videodefs.go new file mode 100644 index 000000000..0afb7cd19 --- /dev/null +++ b/api/bsky/videodefs.go @@ -0,0 +1,22 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.video.defs + +package bsky + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// VideoDefs_JobStatus is a "jobStatus" in the app.bsky.video.defs schema. +type VideoDefs_JobStatus struct { + Blob *lexutil.LexBlob `json:"blob,omitempty" cborgen:"blob,omitempty"` + Did string `json:"did" cborgen:"did"` + Error *string `json:"error,omitempty" cborgen:"error,omitempty"` + JobId string `json:"jobId" cborgen:"jobId"` + Message *string `json:"message,omitempty" cborgen:"message,omitempty"` + // progress: Progress within the current processing state. + Progress *int64 `json:"progress,omitempty" cborgen:"progress,omitempty"` + // state: The state of the video processing job. All values not listed as a known value indicate that the job is in process. + State string `json:"state" cborgen:"state"` +} diff --git a/api/bsky/videogetJobStatus.go b/api/bsky/videogetJobStatus.go new file mode 100644 index 000000000..c35a3dbae --- /dev/null +++ b/api/bsky/videogetJobStatus.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.video.getJobStatus + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// VideoGetJobStatus_Output is the output of a app.bsky.video.getJobStatus call. +type VideoGetJobStatus_Output struct { + JobStatus *VideoDefs_JobStatus `json:"jobStatus" cborgen:"jobStatus"` +} + +// VideoGetJobStatus calls the XRPC method "app.bsky.video.getJobStatus". +func VideoGetJobStatus(ctx context.Context, c lexutil.LexClient, jobId string) (*VideoGetJobStatus_Output, error) { + var out VideoGetJobStatus_Output + + params := map[string]interface{}{} + params["jobId"] = jobId + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.video.getJobStatus", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/videogetUploadLimits.go b/api/bsky/videogetUploadLimits.go new file mode 100644 index 000000000..876718043 --- /dev/null +++ b/api/bsky/videogetUploadLimits.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.video.getUploadLimits + +package bsky + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// VideoGetUploadLimits_Output is the output of a app.bsky.video.getUploadLimits call. +type VideoGetUploadLimits_Output struct { + CanUpload bool `json:"canUpload" cborgen:"canUpload"` + Error *string `json:"error,omitempty" cborgen:"error,omitempty"` + Message *string `json:"message,omitempty" cborgen:"message,omitempty"` + RemainingDailyBytes *int64 `json:"remainingDailyBytes,omitempty" cborgen:"remainingDailyBytes,omitempty"` + RemainingDailyVideos *int64 `json:"remainingDailyVideos,omitempty" cborgen:"remainingDailyVideos,omitempty"` +} + +// VideoGetUploadLimits calls the XRPC method "app.bsky.video.getUploadLimits". +func VideoGetUploadLimits(ctx context.Context, c lexutil.LexClient) (*VideoGetUploadLimits_Output, error) { + var out VideoGetUploadLimits_Output + if err := c.LexDo(ctx, lexutil.Query, "", "app.bsky.video.getUploadLimits", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/videouploadVideo.go b/api/bsky/videouploadVideo.go new file mode 100644 index 000000000..4e7d2e648 --- /dev/null +++ b/api/bsky/videouploadVideo.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: app.bsky.video.uploadVideo + +package bsky + +import ( + "context" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// VideoUploadVideo_Output is the output of a app.bsky.video.uploadVideo call. +type VideoUploadVideo_Output struct { + JobStatus *VideoDefs_JobStatus `json:"jobStatus" cborgen:"jobStatus"` +} + +// VideoUploadVideo calls the XRPC method "app.bsky.video.uploadVideo". +func VideoUploadVideo(ctx context.Context, c lexutil.LexClient, input io.Reader) (*VideoUploadVideo_Output, error) { + var out VideoUploadVideo_Output + if err := c.LexDo(ctx, lexutil.Procedure, "video/mp4", "app.bsky.video.uploadVideo", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/cbor_gen.go b/api/cbor_gen.go deleted file mode 100644 index eb01bcc09..000000000 --- a/api/cbor_gen.go +++ /dev/null @@ -1,1161 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package api - -import ( - "fmt" - "io" - "math" - "sort" - - cid "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf -var _ = cid.Undef -var _ = math.E -var _ = sort.Sort - -func (t *PostRecord) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{165}); err != nil { - return err - } - - // t.Text (string) (string) - if len("text") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"text\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("text"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("text")); err != nil { - return err - } - - if len(t.Text) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Text was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Text))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Text)); err != nil { - return err - } - - // t.Type (string) (string) - if len("$type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"$type\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("$type")); err != nil { - return err - } - - if len(t.Type) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Type was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Type))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Type)); err != nil { - return err - } - - // t.Reply (api.ReplyRef) (struct) - if len("reply") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"reply\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("reply"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("reply")); err != nil { - return err - } - - if err := t.Reply.MarshalCBOR(cw); err != nil { - return err - } - - // t.Entities ([]*api.PostEntity) (slice) - if len("entities") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"entities\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("entities"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("entities")); err != nil { - return err - } - - if len(t.Entities) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field t.Entities was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Entities))); err != nil { - return err - } - for _, v := range t.Entities { - if err := v.MarshalCBOR(cw); err != nil { - return err - } - } - - // t.CreatedAt (string) (string) - if len("createdAt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"createdAt\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("createdAt"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("createdAt")); err != nil { - return err - } - - if len(t.CreatedAt) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.CreatedAt was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.CreatedAt)); err != nil { - return err - } - return nil -} - -func (t *PostRecord) UnmarshalCBOR(r io.Reader) (err error) { - *t = PostRecord{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("PostRecord: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Text (string) (string) - case "text": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Text = string(sval) - } - // t.Type (string) (string) - case "$type": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Type = string(sval) - } - // t.Reply (api.ReplyRef) (struct) - case "reply": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Reply = new(ReplyRef) - if err := t.Reply.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Reply pointer: %w", err) - } - } - - } - // t.Entities ([]*api.PostEntity) (slice) - case "entities": - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - - if extra > cbg.MaxLength { - return fmt.Errorf("t.Entities: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - - if extra > 0 { - t.Entities = make([]*PostEntity, extra) - } - - for i := 0; i < int(extra); i++ { - - var v PostEntity - if err := v.UnmarshalCBOR(cr); err != nil { - return err - } - - t.Entities[i] = &v - } - - // t.CreatedAt (string) (string) - case "createdAt": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.CreatedAt = string(sval) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *PostEntity) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{163}); err != nil { - return err - } - - // t.Type (string) (string) - if len("type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"type\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("type"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("type")); err != nil { - return err - } - - if len(t.Type) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Type was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Type))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Type)); err != nil { - return err - } - - // t.Index (api.TextSlice) (struct) - if len("index") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"index\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("index"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("index")); err != nil { - return err - } - - if err := t.Index.MarshalCBOR(cw); err != nil { - return err - } - - // t.Value (string) (string) - if len("value") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"value\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("value"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("value")); err != nil { - return err - } - - if len(t.Value) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Value was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Value)); err != nil { - return err - } - return nil -} - -func (t *PostEntity) UnmarshalCBOR(r io.Reader) (err error) { - *t = PostEntity{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("PostEntity: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Type (string) (string) - case "type": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Type = string(sval) - } - // t.Index (api.TextSlice) (struct) - case "index": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.Index = new(TextSlice) - if err := t.Index.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Index pointer: %w", err) - } - } - - } - // t.Value (string) (string) - case "value": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Value = string(sval) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *PostRef) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.Cid (string) (string) - if len("Cid") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Cid\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Cid"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Cid")); err != nil { - return err - } - - if len(t.Cid) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Cid was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Cid))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Cid)); err != nil { - return err - } - - // t.Uri (string) (string) - if len("Uri") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Uri\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Uri"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("Uri")); err != nil { - return err - } - - if len(t.Uri) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Uri was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Uri)); err != nil { - return err - } - return nil -} - -func (t *PostRef) UnmarshalCBOR(r io.Reader) (err error) { - *t = PostRef{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("PostRef: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Cid (string) (string) - case "Cid": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Cid = string(sval) - } - // t.Uri (string) (string) - case "Uri": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Uri = string(sval) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *ReplyRef) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.Root (api.PostRef) (struct) - if len("root") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"root\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("root"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("root")); err != nil { - return err - } - - if err := t.Root.MarshalCBOR(cw); err != nil { - return err - } - - // t.Parent (api.PostRef) (struct) - if len("parent") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"parent\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("parent"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("parent")); err != nil { - return err - } - - if err := t.Parent.MarshalCBOR(cw); err != nil { - return err - } - return nil -} - -func (t *ReplyRef) UnmarshalCBOR(r io.Reader) (err error) { - *t = ReplyRef{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("ReplyRef: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Root (api.PostRef) (struct) - case "root": - - { - - if err := t.Root.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Root: %w", err) - } - - } - // t.Parent (api.PostRef) (struct) - case "parent": - - { - - if err := t.Parent.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Parent: %w", err) - } - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *TextSlice) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.End (int64) (int64) - if len("end") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"end\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("end"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("end")); err != nil { - return err - } - - if t.End >= 0 { - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.End)); err != nil { - return err - } - } else { - if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.End-1)); err != nil { - return err - } - } - - // t.Start (int64) (int64) - if len("start") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"start\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("start"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("start")); err != nil { - return err - } - - if t.Start >= 0 { - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Start)); err != nil { - return err - } - } else { - if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Start-1)); err != nil { - return err - } - } - return nil -} - -func (t *TextSlice) UnmarshalCBOR(r io.Reader) (err error) { - *t = TextSlice{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("TextSlice: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.End (int64) (int64) - case "end": - { - maj, extra, err := cr.ReadHeader() - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative overflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.End = int64(extraI) - } - // t.Start (int64) (int64) - case "start": - { - maj, extra, err := cr.ReadHeader() - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative overflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.Start = int64(extraI) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *CreateOp) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - fieldCount := 7 - - if t.Sig == "" { - fieldCount-- - } - - if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { - return err - } - - // t.Sig (string) (string) - if t.Sig != "" { - - if len("sig") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"sig\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sig"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("sig")); err != nil { - return err - } - - if len(t.Sig) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Sig was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Sig))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Sig)); err != nil { - return err - } - } - - // t.Prev (string) (string) - if len("prev") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"prev\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("prev"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("prev")); err != nil { - return err - } - - if t.Prev == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.Prev) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Prev was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Prev))); err != nil { - return err - } - if _, err := io.WriteString(w, string(*t.Prev)); err != nil { - return err - } - } - - // t.Type (string) (string) - if len("type") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"type\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("type"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("type")); err != nil { - return err - } - - if len(t.Type) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Type was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Type))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Type)); err != nil { - return err - } - - // t.Handle (string) (string) - if len("handle") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"handle\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("handle"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("handle")); err != nil { - return err - } - - if len(t.Handle) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Handle was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Handle))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Handle)); err != nil { - return err - } - - // t.Service (string) (string) - if len("service") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"service\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("service"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("service")); err != nil { - return err - } - - if len(t.Service) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.Service was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Service))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.Service)); err != nil { - return err - } - - // t.SigningKey (string) (string) - if len("signingKey") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"signingKey\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("signingKey"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("signingKey")); err != nil { - return err - } - - if len(t.SigningKey) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.SigningKey was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.SigningKey))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.SigningKey)); err != nil { - return err - } - - // t.RecoveryKey (string) (string) - if len("recoveryKey") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"recoveryKey\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("recoveryKey"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("recoveryKey")); err != nil { - return err - } - - if len(t.RecoveryKey) > cbg.MaxLength { - return xerrors.Errorf("Value in field t.RecoveryKey was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.RecoveryKey))); err != nil { - return err - } - if _, err := io.WriteString(w, string(t.RecoveryKey)); err != nil { - return err - } - return nil -} - -func (t *CreateOp) UnmarshalCBOR(r io.Reader) (err error) { - *t = CreateOp{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("CreateOp: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Sig (string) (string) - case "sig": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Sig = string(sval) - } - // t.Prev (string) (string) - case "prev": - - { - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Prev = (*string)(&sval) - } - } - // t.Type (string) (string) - case "type": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Type = string(sval) - } - // t.Handle (string) (string) - case "handle": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Handle = string(sval) - } - // t.Service (string) (string) - case "service": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.Service = string(sval) - } - // t.SigningKey (string) (string) - case "signingKey": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.SigningKey = string(sval) - } - // t.RecoveryKey (string) (string) - case "recoveryKey": - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - t.RecoveryKey = string(sval) - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} diff --git a/api/chat/actordeclaration.go b/api/chat/actordeclaration.go new file mode 100644 index 000000000..6766f6667 --- /dev/null +++ b/api/chat/actordeclaration.go @@ -0,0 +1,18 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.actor.declaration + +package chat + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + lexutil.RegisterType("chat.bsky.actor.declaration", &ActorDeclaration{}) +} + +type ActorDeclaration struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.actor.declaration"` + AllowIncoming string `json:"allowIncoming" cborgen:"allowIncoming"` +} diff --git a/api/chat/actordefs.go b/api/chat/actordefs.go new file mode 100644 index 000000000..547773f39 --- /dev/null +++ b/api/chat/actordefs.go @@ -0,0 +1,24 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.actor.defs + +package chat + +import ( + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" +) + +// ActorDefs_ProfileViewBasic is a "profileViewBasic" in the chat.bsky.actor.defs schema. +type ActorDefs_ProfileViewBasic struct { + Associated *appbsky.ActorDefs_ProfileAssociated `json:"associated,omitempty" cborgen:"associated,omitempty"` + Avatar *string `json:"avatar,omitempty" cborgen:"avatar,omitempty"` + // chatDisabled: Set to true when the actor cannot actively participate in conversations + ChatDisabled *bool `json:"chatDisabled,omitempty" cborgen:"chatDisabled,omitempty"` + Did string `json:"did" cborgen:"did"` + DisplayName *string `json:"displayName,omitempty" cborgen:"displayName,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + Verification *appbsky.ActorDefs_VerificationState `json:"verification,omitempty" cborgen:"verification,omitempty"` + Viewer *appbsky.ActorDefs_ViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} diff --git a/api/chat/actordeleteAccount.go b/api/chat/actordeleteAccount.go new file mode 100644 index 000000000..d59648934 --- /dev/null +++ b/api/chat/actordeleteAccount.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.actor.deleteAccount + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ActorDeleteAccount_Output is the output of a chat.bsky.actor.deleteAccount call. +type ActorDeleteAccount_Output struct { +} + +// ActorDeleteAccount calls the XRPC method "chat.bsky.actor.deleteAccount". +func ActorDeleteAccount(ctx context.Context, c lexutil.LexClient) (*ActorDeleteAccount_Output, error) { + var out ActorDeleteAccount_Output + if err := c.LexDo(ctx, lexutil.Procedure, "", "chat.bsky.actor.deleteAccount", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/actorexportAccountData.go b/api/chat/actorexportAccountData.go new file mode 100644 index 000000000..0851203c1 --- /dev/null +++ b/api/chat/actorexportAccountData.go @@ -0,0 +1,22 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.actor.exportAccountData + +package chat + +import ( + "bytes" + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ActorExportAccountData calls the XRPC method "chat.bsky.actor.exportAccountData". +func ActorExportAccountData(ctx context.Context, c lexutil.LexClient) ([]byte, error) { + buf := new(bytes.Buffer) + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.actor.exportAccountData", nil, nil, buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/api/chat/cbor_gen.go b/api/chat/cbor_gen.go new file mode 100644 index 000000000..44d077e75 --- /dev/null +++ b/api/chat/cbor_gen.go @@ -0,0 +1,150 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package chat + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *ActorDeclaration) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("chat.bsky.actor.declaration"))); err != nil { + return err + } + if _, err := cw.WriteString(string("chat.bsky.actor.declaration")); err != nil { + return err + } + + // t.AllowIncoming (string) (string) + if len("allowIncoming") > 1000000 { + return xerrors.Errorf("Value in field \"allowIncoming\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("allowIncoming"))); err != nil { + return err + } + if _, err := cw.WriteString(string("allowIncoming")); err != nil { + return err + } + + if len(t.AllowIncoming) > 1000000 { + return xerrors.Errorf("Value in field t.AllowIncoming was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.AllowIncoming))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.AllowIncoming)); err != nil { + return err + } + return nil +} + +func (t *ActorDeclaration) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorDeclaration{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorDeclaration: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 13) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.AllowIncoming (string) (string) + case "allowIncoming": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.AllowIncoming = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} diff --git a/api/chat/convoacceptConvo.go b/api/chat/convoacceptConvo.go new file mode 100644 index 000000000..27efe1d96 --- /dev/null +++ b/api/chat/convoacceptConvo.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.acceptConvo + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoAcceptConvo_Input is the input argument to a chat.bsky.convo.acceptConvo call. +type ConvoAcceptConvo_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` +} + +// ConvoAcceptConvo_Output is the output of a chat.bsky.convo.acceptConvo call. +type ConvoAcceptConvo_Output struct { + // rev: Rev when the convo was accepted. If not present, the convo was already accepted. + Rev *string `json:"rev,omitempty" cborgen:"rev,omitempty"` +} + +// ConvoAcceptConvo calls the XRPC method "chat.bsky.convo.acceptConvo". +func ConvoAcceptConvo(ctx context.Context, c lexutil.LexClient, input *ConvoAcceptConvo_Input) (*ConvoAcceptConvo_Output, error) { + var out ConvoAcceptConvo_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.acceptConvo", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convoaddReaction.go b/api/chat/convoaddReaction.go new file mode 100644 index 000000000..296901f69 --- /dev/null +++ b/api/chat/convoaddReaction.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.addReaction + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoAddReaction_Input is the input argument to a chat.bsky.convo.addReaction call. +type ConvoAddReaction_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` + MessageId string `json:"messageId" cborgen:"messageId"` + Value string `json:"value" cborgen:"value"` +} + +// ConvoAddReaction_Output is the output of a chat.bsky.convo.addReaction call. +type ConvoAddReaction_Output struct { + Message *ConvoDefs_MessageView `json:"message" cborgen:"message"` +} + +// ConvoAddReaction calls the XRPC method "chat.bsky.convo.addReaction". +func ConvoAddReaction(ctx context.Context, c lexutil.LexClient, input *ConvoAddReaction_Input) (*ConvoAddReaction_Output, error) { + var out ConvoAddReaction_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.addReaction", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convodefs.go b/api/chat/convodefs.go new file mode 100644 index 000000000..b013ce963 --- /dev/null +++ b/api/chat/convodefs.go @@ -0,0 +1,457 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.defs + +package chat + +import ( + "encoding/json" + "fmt" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoDefs_ConvoView is a "convoView" in the chat.bsky.convo.defs schema. +type ConvoDefs_ConvoView struct { + Id string `json:"id" cborgen:"id"` + LastMessage *ConvoDefs_ConvoView_LastMessage `json:"lastMessage,omitempty" cborgen:"lastMessage,omitempty"` + LastReaction *ConvoDefs_ConvoView_LastReaction `json:"lastReaction,omitempty" cborgen:"lastReaction,omitempty"` + Members []*ActorDefs_ProfileViewBasic `json:"members" cborgen:"members"` + Muted bool `json:"muted" cborgen:"muted"` + Rev string `json:"rev" cborgen:"rev"` + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` + UnreadCount int64 `json:"unreadCount" cborgen:"unreadCount"` +} + +type ConvoDefs_ConvoView_LastMessage struct { + ConvoDefs_MessageView *ConvoDefs_MessageView + ConvoDefs_DeletedMessageView *ConvoDefs_DeletedMessageView +} + +func (t *ConvoDefs_ConvoView_LastMessage) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageView != nil { + t.ConvoDefs_MessageView.LexiconTypeID = "chat.bsky.convo.defs#messageView" + return json.Marshal(t.ConvoDefs_MessageView) + } + if t.ConvoDefs_DeletedMessageView != nil { + t.ConvoDefs_DeletedMessageView.LexiconTypeID = "chat.bsky.convo.defs#deletedMessageView" + return json.Marshal(t.ConvoDefs_DeletedMessageView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_ConvoView_LastMessage) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageView": + t.ConvoDefs_MessageView = new(ConvoDefs_MessageView) + return json.Unmarshal(b, t.ConvoDefs_MessageView) + case "chat.bsky.convo.defs#deletedMessageView": + t.ConvoDefs_DeletedMessageView = new(ConvoDefs_DeletedMessageView) + return json.Unmarshal(b, t.ConvoDefs_DeletedMessageView) + default: + return nil + } +} + +type ConvoDefs_ConvoView_LastReaction struct { + ConvoDefs_MessageAndReactionView *ConvoDefs_MessageAndReactionView +} + +func (t *ConvoDefs_ConvoView_LastReaction) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageAndReactionView != nil { + t.ConvoDefs_MessageAndReactionView.LexiconTypeID = "chat.bsky.convo.defs#messageAndReactionView" + return json.Marshal(t.ConvoDefs_MessageAndReactionView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_ConvoView_LastReaction) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageAndReactionView": + t.ConvoDefs_MessageAndReactionView = new(ConvoDefs_MessageAndReactionView) + return json.Unmarshal(b, t.ConvoDefs_MessageAndReactionView) + default: + return nil + } +} + +// ConvoDefs_DeletedMessageView is a "deletedMessageView" in the chat.bsky.convo.defs schema. +type ConvoDefs_DeletedMessageView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#deletedMessageView"` + Id string `json:"id" cborgen:"id"` + Rev string `json:"rev" cborgen:"rev"` + Sender *ConvoDefs_MessageViewSender `json:"sender" cborgen:"sender"` + SentAt string `json:"sentAt" cborgen:"sentAt"` +} + +// ConvoDefs_LogAcceptConvo is a "logAcceptConvo" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogAcceptConvo struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logAcceptConvo"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Rev string `json:"rev" cborgen:"rev"` +} + +// ConvoDefs_LogAddReaction is a "logAddReaction" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogAddReaction struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logAddReaction"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Message *ConvoDefs_LogAddReaction_Message `json:"message" cborgen:"message"` + Reaction *ConvoDefs_ReactionView `json:"reaction" cborgen:"reaction"` + Rev string `json:"rev" cborgen:"rev"` +} + +type ConvoDefs_LogAddReaction_Message struct { + ConvoDefs_MessageView *ConvoDefs_MessageView + ConvoDefs_DeletedMessageView *ConvoDefs_DeletedMessageView +} + +func (t *ConvoDefs_LogAddReaction_Message) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageView != nil { + t.ConvoDefs_MessageView.LexiconTypeID = "chat.bsky.convo.defs#messageView" + return json.Marshal(t.ConvoDefs_MessageView) + } + if t.ConvoDefs_DeletedMessageView != nil { + t.ConvoDefs_DeletedMessageView.LexiconTypeID = "chat.bsky.convo.defs#deletedMessageView" + return json.Marshal(t.ConvoDefs_DeletedMessageView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_LogAddReaction_Message) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageView": + t.ConvoDefs_MessageView = new(ConvoDefs_MessageView) + return json.Unmarshal(b, t.ConvoDefs_MessageView) + case "chat.bsky.convo.defs#deletedMessageView": + t.ConvoDefs_DeletedMessageView = new(ConvoDefs_DeletedMessageView) + return json.Unmarshal(b, t.ConvoDefs_DeletedMessageView) + default: + return nil + } +} + +// ConvoDefs_LogBeginConvo is a "logBeginConvo" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogBeginConvo struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logBeginConvo"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Rev string `json:"rev" cborgen:"rev"` +} + +// ConvoDefs_LogCreateMessage is a "logCreateMessage" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogCreateMessage struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logCreateMessage"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Message *ConvoDefs_LogCreateMessage_Message `json:"message" cborgen:"message"` + Rev string `json:"rev" cborgen:"rev"` +} + +type ConvoDefs_LogCreateMessage_Message struct { + ConvoDefs_MessageView *ConvoDefs_MessageView + ConvoDefs_DeletedMessageView *ConvoDefs_DeletedMessageView +} + +func (t *ConvoDefs_LogCreateMessage_Message) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageView != nil { + t.ConvoDefs_MessageView.LexiconTypeID = "chat.bsky.convo.defs#messageView" + return json.Marshal(t.ConvoDefs_MessageView) + } + if t.ConvoDefs_DeletedMessageView != nil { + t.ConvoDefs_DeletedMessageView.LexiconTypeID = "chat.bsky.convo.defs#deletedMessageView" + return json.Marshal(t.ConvoDefs_DeletedMessageView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_LogCreateMessage_Message) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageView": + t.ConvoDefs_MessageView = new(ConvoDefs_MessageView) + return json.Unmarshal(b, t.ConvoDefs_MessageView) + case "chat.bsky.convo.defs#deletedMessageView": + t.ConvoDefs_DeletedMessageView = new(ConvoDefs_DeletedMessageView) + return json.Unmarshal(b, t.ConvoDefs_DeletedMessageView) + default: + return nil + } +} + +// ConvoDefs_LogDeleteMessage is a "logDeleteMessage" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogDeleteMessage struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logDeleteMessage"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Message *ConvoDefs_LogDeleteMessage_Message `json:"message" cborgen:"message"` + Rev string `json:"rev" cborgen:"rev"` +} + +type ConvoDefs_LogDeleteMessage_Message struct { + ConvoDefs_MessageView *ConvoDefs_MessageView + ConvoDefs_DeletedMessageView *ConvoDefs_DeletedMessageView +} + +func (t *ConvoDefs_LogDeleteMessage_Message) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageView != nil { + t.ConvoDefs_MessageView.LexiconTypeID = "chat.bsky.convo.defs#messageView" + return json.Marshal(t.ConvoDefs_MessageView) + } + if t.ConvoDefs_DeletedMessageView != nil { + t.ConvoDefs_DeletedMessageView.LexiconTypeID = "chat.bsky.convo.defs#deletedMessageView" + return json.Marshal(t.ConvoDefs_DeletedMessageView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_LogDeleteMessage_Message) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageView": + t.ConvoDefs_MessageView = new(ConvoDefs_MessageView) + return json.Unmarshal(b, t.ConvoDefs_MessageView) + case "chat.bsky.convo.defs#deletedMessageView": + t.ConvoDefs_DeletedMessageView = new(ConvoDefs_DeletedMessageView) + return json.Unmarshal(b, t.ConvoDefs_DeletedMessageView) + default: + return nil + } +} + +// ConvoDefs_LogLeaveConvo is a "logLeaveConvo" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogLeaveConvo struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logLeaveConvo"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Rev string `json:"rev" cborgen:"rev"` +} + +// ConvoDefs_LogMuteConvo is a "logMuteConvo" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogMuteConvo struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logMuteConvo"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Rev string `json:"rev" cborgen:"rev"` +} + +// ConvoDefs_LogReadMessage is a "logReadMessage" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogReadMessage struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logReadMessage"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Message *ConvoDefs_LogReadMessage_Message `json:"message" cborgen:"message"` + Rev string `json:"rev" cborgen:"rev"` +} + +type ConvoDefs_LogReadMessage_Message struct { + ConvoDefs_MessageView *ConvoDefs_MessageView + ConvoDefs_DeletedMessageView *ConvoDefs_DeletedMessageView +} + +func (t *ConvoDefs_LogReadMessage_Message) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageView != nil { + t.ConvoDefs_MessageView.LexiconTypeID = "chat.bsky.convo.defs#messageView" + return json.Marshal(t.ConvoDefs_MessageView) + } + if t.ConvoDefs_DeletedMessageView != nil { + t.ConvoDefs_DeletedMessageView.LexiconTypeID = "chat.bsky.convo.defs#deletedMessageView" + return json.Marshal(t.ConvoDefs_DeletedMessageView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_LogReadMessage_Message) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageView": + t.ConvoDefs_MessageView = new(ConvoDefs_MessageView) + return json.Unmarshal(b, t.ConvoDefs_MessageView) + case "chat.bsky.convo.defs#deletedMessageView": + t.ConvoDefs_DeletedMessageView = new(ConvoDefs_DeletedMessageView) + return json.Unmarshal(b, t.ConvoDefs_DeletedMessageView) + default: + return nil + } +} + +// ConvoDefs_LogRemoveReaction is a "logRemoveReaction" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogRemoveReaction struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logRemoveReaction"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Message *ConvoDefs_LogRemoveReaction_Message `json:"message" cborgen:"message"` + Reaction *ConvoDefs_ReactionView `json:"reaction" cborgen:"reaction"` + Rev string `json:"rev" cborgen:"rev"` +} + +type ConvoDefs_LogRemoveReaction_Message struct { + ConvoDefs_MessageView *ConvoDefs_MessageView + ConvoDefs_DeletedMessageView *ConvoDefs_DeletedMessageView +} + +func (t *ConvoDefs_LogRemoveReaction_Message) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageView != nil { + t.ConvoDefs_MessageView.LexiconTypeID = "chat.bsky.convo.defs#messageView" + return json.Marshal(t.ConvoDefs_MessageView) + } + if t.ConvoDefs_DeletedMessageView != nil { + t.ConvoDefs_DeletedMessageView.LexiconTypeID = "chat.bsky.convo.defs#deletedMessageView" + return json.Marshal(t.ConvoDefs_DeletedMessageView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_LogRemoveReaction_Message) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageView": + t.ConvoDefs_MessageView = new(ConvoDefs_MessageView) + return json.Unmarshal(b, t.ConvoDefs_MessageView) + case "chat.bsky.convo.defs#deletedMessageView": + t.ConvoDefs_DeletedMessageView = new(ConvoDefs_DeletedMessageView) + return json.Unmarshal(b, t.ConvoDefs_DeletedMessageView) + default: + return nil + } +} + +// ConvoDefs_LogUnmuteConvo is a "logUnmuteConvo" in the chat.bsky.convo.defs schema. +type ConvoDefs_LogUnmuteConvo struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#logUnmuteConvo"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Rev string `json:"rev" cborgen:"rev"` +} + +// ConvoDefs_MessageAndReactionView is a "messageAndReactionView" in the chat.bsky.convo.defs schema. +type ConvoDefs_MessageAndReactionView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#messageAndReactionView"` + Message *ConvoDefs_MessageView `json:"message" cborgen:"message"` + Reaction *ConvoDefs_ReactionView `json:"reaction" cborgen:"reaction"` +} + +// ConvoDefs_MessageInput is the input argument to a chat.bsky.convo.defs call. +type ConvoDefs_MessageInput struct { + Embed *ConvoDefs_MessageInput_Embed `json:"embed,omitempty" cborgen:"embed,omitempty"` + // facets: Annotations of text (mentions, URLs, hashtags, etc) + Facets []*appbsky.RichtextFacet `json:"facets,omitempty" cborgen:"facets,omitempty"` + Text string `json:"text" cborgen:"text"` +} + +type ConvoDefs_MessageInput_Embed struct { + EmbedRecord *appbsky.EmbedRecord +} + +func (t *ConvoDefs_MessageInput_Embed) MarshalJSON() ([]byte, error) { + if t.EmbedRecord != nil { + t.EmbedRecord.LexiconTypeID = "app.bsky.embed.record" + return json.Marshal(t.EmbedRecord) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_MessageInput_Embed) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.record": + t.EmbedRecord = new(appbsky.EmbedRecord) + return json.Unmarshal(b, t.EmbedRecord) + default: + return nil + } +} + +// ConvoDefs_MessageRef is a "messageRef" in the chat.bsky.convo.defs schema. +type ConvoDefs_MessageRef struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#messageRef"` + ConvoId string `json:"convoId" cborgen:"convoId"` + Did string `json:"did" cborgen:"did"` + MessageId string `json:"messageId" cborgen:"messageId"` +} + +// ConvoDefs_MessageView is a "messageView" in the chat.bsky.convo.defs schema. +type ConvoDefs_MessageView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=chat.bsky.convo.defs#messageView"` + Embed *ConvoDefs_MessageView_Embed `json:"embed,omitempty" cborgen:"embed,omitempty"` + // facets: Annotations of text (mentions, URLs, hashtags, etc) + Facets []*appbsky.RichtextFacet `json:"facets,omitempty" cborgen:"facets,omitempty"` + Id string `json:"id" cborgen:"id"` + // reactions: Reactions to this message, in ascending order of creation time. + Reactions []*ConvoDefs_ReactionView `json:"reactions,omitempty" cborgen:"reactions,omitempty"` + Rev string `json:"rev" cborgen:"rev"` + Sender *ConvoDefs_MessageViewSender `json:"sender" cborgen:"sender"` + SentAt string `json:"sentAt" cborgen:"sentAt"` + Text string `json:"text" cborgen:"text"` +} + +// ConvoDefs_MessageViewSender is a "messageViewSender" in the chat.bsky.convo.defs schema. +type ConvoDefs_MessageViewSender struct { + Did string `json:"did" cborgen:"did"` +} + +type ConvoDefs_MessageView_Embed struct { + EmbedRecord_View *appbsky.EmbedRecord_View +} + +func (t *ConvoDefs_MessageView_Embed) MarshalJSON() ([]byte, error) { + if t.EmbedRecord_View != nil { + t.EmbedRecord_View.LexiconTypeID = "app.bsky.embed.record#view" + return json.Marshal(t.EmbedRecord_View) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoDefs_MessageView_Embed) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.embed.record#view": + t.EmbedRecord_View = new(appbsky.EmbedRecord_View) + return json.Unmarshal(b, t.EmbedRecord_View) + default: + return nil + } +} + +// ConvoDefs_ReactionView is a "reactionView" in the chat.bsky.convo.defs schema. +type ConvoDefs_ReactionView struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Sender *ConvoDefs_ReactionViewSender `json:"sender" cborgen:"sender"` + Value string `json:"value" cborgen:"value"` +} + +// ConvoDefs_ReactionViewSender is a "reactionViewSender" in the chat.bsky.convo.defs schema. +type ConvoDefs_ReactionViewSender struct { + Did string `json:"did" cborgen:"did"` +} diff --git a/api/chat/convodeleteMessageForSelf.go b/api/chat/convodeleteMessageForSelf.go new file mode 100644 index 000000000..b2bb8691f --- /dev/null +++ b/api/chat/convodeleteMessageForSelf.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.deleteMessageForSelf + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoDeleteMessageForSelf_Input is the input argument to a chat.bsky.convo.deleteMessageForSelf call. +type ConvoDeleteMessageForSelf_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` + MessageId string `json:"messageId" cborgen:"messageId"` +} + +// ConvoDeleteMessageForSelf calls the XRPC method "chat.bsky.convo.deleteMessageForSelf". +func ConvoDeleteMessageForSelf(ctx context.Context, c lexutil.LexClient, input *ConvoDeleteMessageForSelf_Input) (*ConvoDefs_DeletedMessageView, error) { + var out ConvoDefs_DeletedMessageView + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.deleteMessageForSelf", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convogetConvo.go b/api/chat/convogetConvo.go new file mode 100644 index 000000000..e87a2e030 --- /dev/null +++ b/api/chat/convogetConvo.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.getConvo + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoGetConvo_Output is the output of a chat.bsky.convo.getConvo call. +type ConvoGetConvo_Output struct { + Convo *ConvoDefs_ConvoView `json:"convo" cborgen:"convo"` +} + +// ConvoGetConvo calls the XRPC method "chat.bsky.convo.getConvo". +func ConvoGetConvo(ctx context.Context, c lexutil.LexClient, convoId string) (*ConvoGetConvo_Output, error) { + var out ConvoGetConvo_Output + + params := map[string]interface{}{} + params["convoId"] = convoId + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.convo.getConvo", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convogetConvoAvailability.go b/api/chat/convogetConvoAvailability.go new file mode 100644 index 000000000..670fe268c --- /dev/null +++ b/api/chat/convogetConvoAvailability.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.getConvoAvailability + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoGetConvoAvailability_Output is the output of a chat.bsky.convo.getConvoAvailability call. +type ConvoGetConvoAvailability_Output struct { + CanChat bool `json:"canChat" cborgen:"canChat"` + Convo *ConvoDefs_ConvoView `json:"convo,omitempty" cborgen:"convo,omitempty"` +} + +// ConvoGetConvoAvailability calls the XRPC method "chat.bsky.convo.getConvoAvailability". +func ConvoGetConvoAvailability(ctx context.Context, c lexutil.LexClient, members []string) (*ConvoGetConvoAvailability_Output, error) { + var out ConvoGetConvoAvailability_Output + + params := map[string]interface{}{} + params["members"] = members + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.convo.getConvoAvailability", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convogetConvoForMembers.go b/api/chat/convogetConvoForMembers.go new file mode 100644 index 000000000..2ab28d829 --- /dev/null +++ b/api/chat/convogetConvoForMembers.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.getConvoForMembers + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoGetConvoForMembers_Output is the output of a chat.bsky.convo.getConvoForMembers call. +type ConvoGetConvoForMembers_Output struct { + Convo *ConvoDefs_ConvoView `json:"convo" cborgen:"convo"` +} + +// ConvoGetConvoForMembers calls the XRPC method "chat.bsky.convo.getConvoForMembers". +func ConvoGetConvoForMembers(ctx context.Context, c lexutil.LexClient, members []string) (*ConvoGetConvoForMembers_Output, error) { + var out ConvoGetConvoForMembers_Output + + params := map[string]interface{}{} + params["members"] = members + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.convo.getConvoForMembers", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convogetLog.go b/api/chat/convogetLog.go new file mode 100644 index 000000000..482f553db --- /dev/null +++ b/api/chat/convogetLog.go @@ -0,0 +1,133 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.getLog + +package chat + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoGetLog_Output is the output of a chat.bsky.convo.getLog call. +type ConvoGetLog_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Logs []*ConvoGetLog_Output_Logs_Elem `json:"logs" cborgen:"logs"` +} + +type ConvoGetLog_Output_Logs_Elem struct { + ConvoDefs_LogBeginConvo *ConvoDefs_LogBeginConvo + ConvoDefs_LogAcceptConvo *ConvoDefs_LogAcceptConvo + ConvoDefs_LogLeaveConvo *ConvoDefs_LogLeaveConvo + ConvoDefs_LogMuteConvo *ConvoDefs_LogMuteConvo + ConvoDefs_LogUnmuteConvo *ConvoDefs_LogUnmuteConvo + ConvoDefs_LogCreateMessage *ConvoDefs_LogCreateMessage + ConvoDefs_LogDeleteMessage *ConvoDefs_LogDeleteMessage + ConvoDefs_LogReadMessage *ConvoDefs_LogReadMessage + ConvoDefs_LogAddReaction *ConvoDefs_LogAddReaction + ConvoDefs_LogRemoveReaction *ConvoDefs_LogRemoveReaction +} + +func (t *ConvoGetLog_Output_Logs_Elem) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_LogBeginConvo != nil { + t.ConvoDefs_LogBeginConvo.LexiconTypeID = "chat.bsky.convo.defs#logBeginConvo" + return json.Marshal(t.ConvoDefs_LogBeginConvo) + } + if t.ConvoDefs_LogAcceptConvo != nil { + t.ConvoDefs_LogAcceptConvo.LexiconTypeID = "chat.bsky.convo.defs#logAcceptConvo" + return json.Marshal(t.ConvoDefs_LogAcceptConvo) + } + if t.ConvoDefs_LogLeaveConvo != nil { + t.ConvoDefs_LogLeaveConvo.LexiconTypeID = "chat.bsky.convo.defs#logLeaveConvo" + return json.Marshal(t.ConvoDefs_LogLeaveConvo) + } + if t.ConvoDefs_LogMuteConvo != nil { + t.ConvoDefs_LogMuteConvo.LexiconTypeID = "chat.bsky.convo.defs#logMuteConvo" + return json.Marshal(t.ConvoDefs_LogMuteConvo) + } + if t.ConvoDefs_LogUnmuteConvo != nil { + t.ConvoDefs_LogUnmuteConvo.LexiconTypeID = "chat.bsky.convo.defs#logUnmuteConvo" + return json.Marshal(t.ConvoDefs_LogUnmuteConvo) + } + if t.ConvoDefs_LogCreateMessage != nil { + t.ConvoDefs_LogCreateMessage.LexiconTypeID = "chat.bsky.convo.defs#logCreateMessage" + return json.Marshal(t.ConvoDefs_LogCreateMessage) + } + if t.ConvoDefs_LogDeleteMessage != nil { + t.ConvoDefs_LogDeleteMessage.LexiconTypeID = "chat.bsky.convo.defs#logDeleteMessage" + return json.Marshal(t.ConvoDefs_LogDeleteMessage) + } + if t.ConvoDefs_LogReadMessage != nil { + t.ConvoDefs_LogReadMessage.LexiconTypeID = "chat.bsky.convo.defs#logReadMessage" + return json.Marshal(t.ConvoDefs_LogReadMessage) + } + if t.ConvoDefs_LogAddReaction != nil { + t.ConvoDefs_LogAddReaction.LexiconTypeID = "chat.bsky.convo.defs#logAddReaction" + return json.Marshal(t.ConvoDefs_LogAddReaction) + } + if t.ConvoDefs_LogRemoveReaction != nil { + t.ConvoDefs_LogRemoveReaction.LexiconTypeID = "chat.bsky.convo.defs#logRemoveReaction" + return json.Marshal(t.ConvoDefs_LogRemoveReaction) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoGetLog_Output_Logs_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#logBeginConvo": + t.ConvoDefs_LogBeginConvo = new(ConvoDefs_LogBeginConvo) + return json.Unmarshal(b, t.ConvoDefs_LogBeginConvo) + case "chat.bsky.convo.defs#logAcceptConvo": + t.ConvoDefs_LogAcceptConvo = new(ConvoDefs_LogAcceptConvo) + return json.Unmarshal(b, t.ConvoDefs_LogAcceptConvo) + case "chat.bsky.convo.defs#logLeaveConvo": + t.ConvoDefs_LogLeaveConvo = new(ConvoDefs_LogLeaveConvo) + return json.Unmarshal(b, t.ConvoDefs_LogLeaveConvo) + case "chat.bsky.convo.defs#logMuteConvo": + t.ConvoDefs_LogMuteConvo = new(ConvoDefs_LogMuteConvo) + return json.Unmarshal(b, t.ConvoDefs_LogMuteConvo) + case "chat.bsky.convo.defs#logUnmuteConvo": + t.ConvoDefs_LogUnmuteConvo = new(ConvoDefs_LogUnmuteConvo) + return json.Unmarshal(b, t.ConvoDefs_LogUnmuteConvo) + case "chat.bsky.convo.defs#logCreateMessage": + t.ConvoDefs_LogCreateMessage = new(ConvoDefs_LogCreateMessage) + return json.Unmarshal(b, t.ConvoDefs_LogCreateMessage) + case "chat.bsky.convo.defs#logDeleteMessage": + t.ConvoDefs_LogDeleteMessage = new(ConvoDefs_LogDeleteMessage) + return json.Unmarshal(b, t.ConvoDefs_LogDeleteMessage) + case "chat.bsky.convo.defs#logReadMessage": + t.ConvoDefs_LogReadMessage = new(ConvoDefs_LogReadMessage) + return json.Unmarshal(b, t.ConvoDefs_LogReadMessage) + case "chat.bsky.convo.defs#logAddReaction": + t.ConvoDefs_LogAddReaction = new(ConvoDefs_LogAddReaction) + return json.Unmarshal(b, t.ConvoDefs_LogAddReaction) + case "chat.bsky.convo.defs#logRemoveReaction": + t.ConvoDefs_LogRemoveReaction = new(ConvoDefs_LogRemoveReaction) + return json.Unmarshal(b, t.ConvoDefs_LogRemoveReaction) + default: + return nil + } +} + +// ConvoGetLog calls the XRPC method "chat.bsky.convo.getLog". +func ConvoGetLog(ctx context.Context, c lexutil.LexClient, cursor string) (*ConvoGetLog_Output, error) { + var out ConvoGetLog_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.convo.getLog", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convogetMessages.go b/api/chat/convogetMessages.go new file mode 100644 index 000000000..f3eb66cb1 --- /dev/null +++ b/api/chat/convogetMessages.go @@ -0,0 +1,73 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.getMessages + +package chat + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoGetMessages_Output is the output of a chat.bsky.convo.getMessages call. +type ConvoGetMessages_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Messages []*ConvoGetMessages_Output_Messages_Elem `json:"messages" cborgen:"messages"` +} + +type ConvoGetMessages_Output_Messages_Elem struct { + ConvoDefs_MessageView *ConvoDefs_MessageView + ConvoDefs_DeletedMessageView *ConvoDefs_DeletedMessageView +} + +func (t *ConvoGetMessages_Output_Messages_Elem) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageView != nil { + t.ConvoDefs_MessageView.LexiconTypeID = "chat.bsky.convo.defs#messageView" + return json.Marshal(t.ConvoDefs_MessageView) + } + if t.ConvoDefs_DeletedMessageView != nil { + t.ConvoDefs_DeletedMessageView.LexiconTypeID = "chat.bsky.convo.defs#deletedMessageView" + return json.Marshal(t.ConvoDefs_DeletedMessageView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ConvoGetMessages_Output_Messages_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageView": + t.ConvoDefs_MessageView = new(ConvoDefs_MessageView) + return json.Unmarshal(b, t.ConvoDefs_MessageView) + case "chat.bsky.convo.defs#deletedMessageView": + t.ConvoDefs_DeletedMessageView = new(ConvoDefs_DeletedMessageView) + return json.Unmarshal(b, t.ConvoDefs_DeletedMessageView) + default: + return nil + } +} + +// ConvoGetMessages calls the XRPC method "chat.bsky.convo.getMessages". +func ConvoGetMessages(ctx context.Context, c lexutil.LexClient, convoId string, cursor string, limit int64) (*ConvoGetMessages_Output, error) { + var out ConvoGetMessages_Output + + params := map[string]interface{}{} + params["convoId"] = convoId + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.convo.getMessages", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convoleaveConvo.go b/api/chat/convoleaveConvo.go new file mode 100644 index 000000000..3429eb494 --- /dev/null +++ b/api/chat/convoleaveConvo.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.leaveConvo + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoLeaveConvo_Input is the input argument to a chat.bsky.convo.leaveConvo call. +type ConvoLeaveConvo_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` +} + +// ConvoLeaveConvo_Output is the output of a chat.bsky.convo.leaveConvo call. +type ConvoLeaveConvo_Output struct { + ConvoId string `json:"convoId" cborgen:"convoId"` + Rev string `json:"rev" cborgen:"rev"` +} + +// ConvoLeaveConvo calls the XRPC method "chat.bsky.convo.leaveConvo". +func ConvoLeaveConvo(ctx context.Context, c lexutil.LexClient, input *ConvoLeaveConvo_Input) (*ConvoLeaveConvo_Output, error) { + var out ConvoLeaveConvo_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.leaveConvo", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convolistConvos.go b/api/chat/convolistConvos.go new file mode 100644 index 000000000..0e516d949 --- /dev/null +++ b/api/chat/convolistConvos.go @@ -0,0 +1,41 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.listConvos + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoListConvos_Output is the output of a chat.bsky.convo.listConvos call. +type ConvoListConvos_Output struct { + Convos []*ConvoDefs_ConvoView `json:"convos" cborgen:"convos"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// ConvoListConvos calls the XRPC method "chat.bsky.convo.listConvos". +func ConvoListConvos(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, readState string, status string) (*ConvoListConvos_Output, error) { + var out ConvoListConvos_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if readState != "" { + params["readState"] = readState + } + if status != "" { + params["status"] = status + } + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.convo.listConvos", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convomuteConvo.go b/api/chat/convomuteConvo.go new file mode 100644 index 000000000..37b6c455a --- /dev/null +++ b/api/chat/convomuteConvo.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.muteConvo + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoMuteConvo_Input is the input argument to a chat.bsky.convo.muteConvo call. +type ConvoMuteConvo_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` +} + +// ConvoMuteConvo_Output is the output of a chat.bsky.convo.muteConvo call. +type ConvoMuteConvo_Output struct { + Convo *ConvoDefs_ConvoView `json:"convo" cborgen:"convo"` +} + +// ConvoMuteConvo calls the XRPC method "chat.bsky.convo.muteConvo". +func ConvoMuteConvo(ctx context.Context, c lexutil.LexClient, input *ConvoMuteConvo_Input) (*ConvoMuteConvo_Output, error) { + var out ConvoMuteConvo_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.muteConvo", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convoremoveReaction.go b/api/chat/convoremoveReaction.go new file mode 100644 index 000000000..f8ec1c701 --- /dev/null +++ b/api/chat/convoremoveReaction.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.removeReaction + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoRemoveReaction_Input is the input argument to a chat.bsky.convo.removeReaction call. +type ConvoRemoveReaction_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` + MessageId string `json:"messageId" cborgen:"messageId"` + Value string `json:"value" cborgen:"value"` +} + +// ConvoRemoveReaction_Output is the output of a chat.bsky.convo.removeReaction call. +type ConvoRemoveReaction_Output struct { + Message *ConvoDefs_MessageView `json:"message" cborgen:"message"` +} + +// ConvoRemoveReaction calls the XRPC method "chat.bsky.convo.removeReaction". +func ConvoRemoveReaction(ctx context.Context, c lexutil.LexClient, input *ConvoRemoveReaction_Input) (*ConvoRemoveReaction_Output, error) { + var out ConvoRemoveReaction_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.removeReaction", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convosendMessage.go b/api/chat/convosendMessage.go new file mode 100644 index 000000000..1f723615b --- /dev/null +++ b/api/chat/convosendMessage.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.sendMessage + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoSendMessage_Input is the input argument to a chat.bsky.convo.sendMessage call. +type ConvoSendMessage_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` + Message *ConvoDefs_MessageInput `json:"message" cborgen:"message"` +} + +// ConvoSendMessage calls the XRPC method "chat.bsky.convo.sendMessage". +func ConvoSendMessage(ctx context.Context, c lexutil.LexClient, input *ConvoSendMessage_Input) (*ConvoDefs_MessageView, error) { + var out ConvoDefs_MessageView + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.sendMessage", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convosendMessageBatch.go b/api/chat/convosendMessageBatch.go new file mode 100644 index 000000000..01cc3f25a --- /dev/null +++ b/api/chat/convosendMessageBatch.go @@ -0,0 +1,37 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.sendMessageBatch + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoSendMessageBatch_BatchItem is a "batchItem" in the chat.bsky.convo.sendMessageBatch schema. +type ConvoSendMessageBatch_BatchItem struct { + ConvoId string `json:"convoId" cborgen:"convoId"` + Message *ConvoDefs_MessageInput `json:"message" cborgen:"message"` +} + +// ConvoSendMessageBatch_Input is the input argument to a chat.bsky.convo.sendMessageBatch call. +type ConvoSendMessageBatch_Input struct { + Items []*ConvoSendMessageBatch_BatchItem `json:"items" cborgen:"items"` +} + +// ConvoSendMessageBatch_Output is the output of a chat.bsky.convo.sendMessageBatch call. +type ConvoSendMessageBatch_Output struct { + Items []*ConvoDefs_MessageView `json:"items" cborgen:"items"` +} + +// ConvoSendMessageBatch calls the XRPC method "chat.bsky.convo.sendMessageBatch". +func ConvoSendMessageBatch(ctx context.Context, c lexutil.LexClient, input *ConvoSendMessageBatch_Input) (*ConvoSendMessageBatch_Output, error) { + var out ConvoSendMessageBatch_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.sendMessageBatch", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convounmuteConvo.go b/api/chat/convounmuteConvo.go new file mode 100644 index 000000000..7228d5497 --- /dev/null +++ b/api/chat/convounmuteConvo.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.unmuteConvo + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoUnmuteConvo_Input is the input argument to a chat.bsky.convo.unmuteConvo call. +type ConvoUnmuteConvo_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` +} + +// ConvoUnmuteConvo_Output is the output of a chat.bsky.convo.unmuteConvo call. +type ConvoUnmuteConvo_Output struct { + Convo *ConvoDefs_ConvoView `json:"convo" cborgen:"convo"` +} + +// ConvoUnmuteConvo calls the XRPC method "chat.bsky.convo.unmuteConvo". +func ConvoUnmuteConvo(ctx context.Context, c lexutil.LexClient, input *ConvoUnmuteConvo_Input) (*ConvoUnmuteConvo_Output, error) { + var out ConvoUnmuteConvo_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.unmuteConvo", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convoupdateAllRead.go b/api/chat/convoupdateAllRead.go new file mode 100644 index 000000000..02d18cd4e --- /dev/null +++ b/api/chat/convoupdateAllRead.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.updateAllRead + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoUpdateAllRead_Input is the input argument to a chat.bsky.convo.updateAllRead call. +type ConvoUpdateAllRead_Input struct { + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +// ConvoUpdateAllRead_Output is the output of a chat.bsky.convo.updateAllRead call. +type ConvoUpdateAllRead_Output struct { + // updatedCount: The count of updated convos. + UpdatedCount int64 `json:"updatedCount" cborgen:"updatedCount"` +} + +// ConvoUpdateAllRead calls the XRPC method "chat.bsky.convo.updateAllRead". +func ConvoUpdateAllRead(ctx context.Context, c lexutil.LexClient, input *ConvoUpdateAllRead_Input) (*ConvoUpdateAllRead_Output, error) { + var out ConvoUpdateAllRead_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.updateAllRead", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/convoupdateRead.go b/api/chat/convoupdateRead.go new file mode 100644 index 000000000..85494fb81 --- /dev/null +++ b/api/chat/convoupdateRead.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.convo.updateRead + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ConvoUpdateRead_Input is the input argument to a chat.bsky.convo.updateRead call. +type ConvoUpdateRead_Input struct { + ConvoId string `json:"convoId" cborgen:"convoId"` + MessageId *string `json:"messageId,omitempty" cborgen:"messageId,omitempty"` +} + +// ConvoUpdateRead_Output is the output of a chat.bsky.convo.updateRead call. +type ConvoUpdateRead_Output struct { + Convo *ConvoDefs_ConvoView `json:"convo" cborgen:"convo"` +} + +// ConvoUpdateRead calls the XRPC method "chat.bsky.convo.updateRead". +func ConvoUpdateRead(ctx context.Context, c lexutil.LexClient, input *ConvoUpdateRead_Input) (*ConvoUpdateRead_Output, error) { + var out ConvoUpdateRead_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.convo.updateRead", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/moderationgetActorMetadata.go b/api/chat/moderationgetActorMetadata.go new file mode 100644 index 000000000..401c7f364 --- /dev/null +++ b/api/chat/moderationgetActorMetadata.go @@ -0,0 +1,39 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.moderation.getActorMetadata + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetActorMetadata_Metadata is a "metadata" in the chat.bsky.moderation.getActorMetadata schema. +type ModerationGetActorMetadata_Metadata struct { + Convos int64 `json:"convos" cborgen:"convos"` + ConvosStarted int64 `json:"convosStarted" cborgen:"convosStarted"` + MessagesReceived int64 `json:"messagesReceived" cborgen:"messagesReceived"` + MessagesSent int64 `json:"messagesSent" cborgen:"messagesSent"` +} + +// ModerationGetActorMetadata_Output is the output of a chat.bsky.moderation.getActorMetadata call. +type ModerationGetActorMetadata_Output struct { + All *ModerationGetActorMetadata_Metadata `json:"all" cborgen:"all"` + Day *ModerationGetActorMetadata_Metadata `json:"day" cborgen:"day"` + Month *ModerationGetActorMetadata_Metadata `json:"month" cborgen:"month"` +} + +// ModerationGetActorMetadata calls the XRPC method "chat.bsky.moderation.getActorMetadata". +func ModerationGetActorMetadata(ctx context.Context, c lexutil.LexClient, actor string) (*ModerationGetActorMetadata_Output, error) { + var out ModerationGetActorMetadata_Output + + params := map[string]interface{}{} + params["actor"] = actor + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.moderation.getActorMetadata", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/moderationgetMessageContext.go b/api/chat/moderationgetMessageContext.go new file mode 100644 index 000000000..764ca273b --- /dev/null +++ b/api/chat/moderationgetMessageContext.go @@ -0,0 +1,77 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.moderation.getMessageContext + +package chat + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetMessageContext_Output is the output of a chat.bsky.moderation.getMessageContext call. +type ModerationGetMessageContext_Output struct { + Messages []*ModerationGetMessageContext_Output_Messages_Elem `json:"messages" cborgen:"messages"` +} + +type ModerationGetMessageContext_Output_Messages_Elem struct { + ConvoDefs_MessageView *ConvoDefs_MessageView + ConvoDefs_DeletedMessageView *ConvoDefs_DeletedMessageView +} + +func (t *ModerationGetMessageContext_Output_Messages_Elem) MarshalJSON() ([]byte, error) { + if t.ConvoDefs_MessageView != nil { + t.ConvoDefs_MessageView.LexiconTypeID = "chat.bsky.convo.defs#messageView" + return json.Marshal(t.ConvoDefs_MessageView) + } + if t.ConvoDefs_DeletedMessageView != nil { + t.ConvoDefs_DeletedMessageView.LexiconTypeID = "chat.bsky.convo.defs#deletedMessageView" + return json.Marshal(t.ConvoDefs_DeletedMessageView) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationGetMessageContext_Output_Messages_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "chat.bsky.convo.defs#messageView": + t.ConvoDefs_MessageView = new(ConvoDefs_MessageView) + return json.Unmarshal(b, t.ConvoDefs_MessageView) + case "chat.bsky.convo.defs#deletedMessageView": + t.ConvoDefs_DeletedMessageView = new(ConvoDefs_DeletedMessageView) + return json.Unmarshal(b, t.ConvoDefs_DeletedMessageView) + default: + return nil + } +} + +// ModerationGetMessageContext calls the XRPC method "chat.bsky.moderation.getMessageContext". +// +// convoId: Conversation that the message is from. NOTE: this field will eventually be required. +func ModerationGetMessageContext(ctx context.Context, c lexutil.LexClient, after int64, before int64, convoId string, messageId string) (*ModerationGetMessageContext_Output, error) { + var out ModerationGetMessageContext_Output + + params := map[string]interface{}{} + if after != 0 { + params["after"] = after + } + if before != 0 { + params["before"] = before + } + if convoId != "" { + params["convoId"] = convoId + } + params["messageId"] = messageId + if err := c.LexDo(ctx, lexutil.Query, "", "chat.bsky.moderation.getMessageContext", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/chat/moderationupdateActorAccess.go b/api/chat/moderationupdateActorAccess.go new file mode 100644 index 000000000..121555fec --- /dev/null +++ b/api/chat/moderationupdateActorAccess.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: chat.bsky.moderation.updateActorAccess + +package chat + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationUpdateActorAccess_Input is the input argument to a chat.bsky.moderation.updateActorAccess call. +type ModerationUpdateActorAccess_Input struct { + Actor string `json:"actor" cborgen:"actor"` + AllowAccess bool `json:"allowAccess" cborgen:"allowAccess"` + Ref *string `json:"ref,omitempty" cborgen:"ref,omitempty"` +} + +// ModerationUpdateActorAccess calls the XRPC method "chat.bsky.moderation.updateActorAccess". +func ModerationUpdateActorAccess(ctx context.Context, c lexutil.LexClient, input *ModerationUpdateActorAccess_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "chat.bsky.moderation.updateActorAccess", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/extra.go b/api/extra.go deleted file mode 100644 index 3b30e0045..000000000 --- a/api/extra.go +++ /dev/null @@ -1,60 +0,0 @@ -package api - -import ( - "context" - "fmt" - "net/url" - - did "github.com/whyrusleeping/go-did" - otel "go.opentelemetry.io/otel" -) - -func ResolveDidToHandle(ctx context.Context, atp *ATProto, pls *PLCServer, udid string) (string, string, error) { - ctx, span := otel.Tracer("gosky").Start(ctx, "resolveDidToHandle") - defer span.End() - - doc, err := pls.GetDocument(ctx, udid) - if err != nil { - return "", "", err - } - - if len(doc.AlsoKnownAs) == 0 { - return "", "", fmt.Errorf("users did document does not specify a handle") - } - - aka := doc.AlsoKnownAs[0] - - u, err := url.Parse(aka) - if err != nil { - return "", "", fmt.Errorf("aka field in doc was not a valid url: %w", err) - } - - handle := u.Host - - var svc *did.Service - for _, s := range doc.Service { - if s.Type == "AtpPersonalDataServer" { - svc = &s - break - } - } - - if svc == nil { - return "", "", fmt.Errorf("users did document has no pds service set") - } - - if svc.ServiceEndpoint != atp.C.Host { - return "", "", fmt.Errorf("our atp client is authed for a different pds (%s != %s)", svc.ServiceEndpoint, atp.C.Host) - } - - verdid, err := atp.HandleResolve(ctx, handle) - if err != nil { - return "", "", err - } - - if verdid != udid { - return "", "", fmt.Errorf("pds server reported different did for claimed handle") - } - - return handle, svc.ServiceEndpoint, nil -} diff --git a/api/ozone/communicationcreateTemplate.go b/api/ozone/communicationcreateTemplate.go new file mode 100644 index 000000000..3a04e59ee --- /dev/null +++ b/api/ozone/communicationcreateTemplate.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.communication.createTemplate + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// CommunicationCreateTemplate_Input is the input argument to a tools.ozone.communication.createTemplate call. +type CommunicationCreateTemplate_Input struct { + // contentMarkdown: Content of the template, markdown supported, can contain variable placeholders. + ContentMarkdown string `json:"contentMarkdown" cborgen:"contentMarkdown"` + // createdBy: DID of the user who is creating the template. + CreatedBy *string `json:"createdBy,omitempty" cborgen:"createdBy,omitempty"` + // lang: Message language. + Lang *string `json:"lang,omitempty" cborgen:"lang,omitempty"` + // name: Name of the template. + Name string `json:"name" cborgen:"name"` + // subject: Subject of the message, used in emails. + Subject string `json:"subject" cborgen:"subject"` +} + +// CommunicationCreateTemplate calls the XRPC method "tools.ozone.communication.createTemplate". +func CommunicationCreateTemplate(ctx context.Context, c lexutil.LexClient, input *CommunicationCreateTemplate_Input) (*CommunicationDefs_TemplateView, error) { + var out CommunicationDefs_TemplateView + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.communication.createTemplate", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/communicationdefs.go b/api/ozone/communicationdefs.go new file mode 100644 index 000000000..b384f2bef --- /dev/null +++ b/api/ozone/communicationdefs.go @@ -0,0 +1,23 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.communication.defs + +package ozone + +// CommunicationDefs_TemplateView is a "templateView" in the tools.ozone.communication.defs schema. +type CommunicationDefs_TemplateView struct { + // contentMarkdown: Subject of the message, used in emails. + ContentMarkdown string `json:"contentMarkdown" cborgen:"contentMarkdown"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Disabled bool `json:"disabled" cborgen:"disabled"` + Id string `json:"id" cborgen:"id"` + // lang: Message language. + Lang *string `json:"lang,omitempty" cborgen:"lang,omitempty"` + // lastUpdatedBy: DID of the user who last updated the template. + LastUpdatedBy string `json:"lastUpdatedBy" cborgen:"lastUpdatedBy"` + // name: Name of the template. + Name string `json:"name" cborgen:"name"` + // subject: Content of the template, can contain markdown and variable placeholders. + Subject *string `json:"subject,omitempty" cborgen:"subject,omitempty"` + UpdatedAt string `json:"updatedAt" cborgen:"updatedAt"` +} diff --git a/api/ozone/communicationdeleteTemplate.go b/api/ozone/communicationdeleteTemplate.go new file mode 100644 index 000000000..0a7f911bd --- /dev/null +++ b/api/ozone/communicationdeleteTemplate.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.communication.deleteTemplate + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// CommunicationDeleteTemplate_Input is the input argument to a tools.ozone.communication.deleteTemplate call. +type CommunicationDeleteTemplate_Input struct { + Id string `json:"id" cborgen:"id"` +} + +// CommunicationDeleteTemplate calls the XRPC method "tools.ozone.communication.deleteTemplate". +func CommunicationDeleteTemplate(ctx context.Context, c lexutil.LexClient, input *CommunicationDeleteTemplate_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.communication.deleteTemplate", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/ozone/communicationlistTemplates.go b/api/ozone/communicationlistTemplates.go new file mode 100644 index 000000000..d95aa3019 --- /dev/null +++ b/api/ozone/communicationlistTemplates.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.communication.listTemplates + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// CommunicationListTemplates_Output is the output of a tools.ozone.communication.listTemplates call. +type CommunicationListTemplates_Output struct { + CommunicationTemplates []*CommunicationDefs_TemplateView `json:"communicationTemplates" cborgen:"communicationTemplates"` +} + +// CommunicationListTemplates calls the XRPC method "tools.ozone.communication.listTemplates". +func CommunicationListTemplates(ctx context.Context, c lexutil.LexClient) (*CommunicationListTemplates_Output, error) { + var out CommunicationListTemplates_Output + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.communication.listTemplates", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/communicationupdateTemplate.go b/api/ozone/communicationupdateTemplate.go new file mode 100644 index 000000000..bded5a725 --- /dev/null +++ b/api/ozone/communicationupdateTemplate.go @@ -0,0 +1,38 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.communication.updateTemplate + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// CommunicationUpdateTemplate_Input is the input argument to a tools.ozone.communication.updateTemplate call. +type CommunicationUpdateTemplate_Input struct { + // contentMarkdown: Content of the template, markdown supported, can contain variable placeholders. + ContentMarkdown *string `json:"contentMarkdown,omitempty" cborgen:"contentMarkdown,omitempty"` + Disabled *bool `json:"disabled,omitempty" cborgen:"disabled,omitempty"` + // id: ID of the template to be updated. + Id string `json:"id" cborgen:"id"` + // lang: Message language. + Lang *string `json:"lang,omitempty" cborgen:"lang,omitempty"` + // name: Name of the template. + Name *string `json:"name,omitempty" cborgen:"name,omitempty"` + // subject: Subject of the message, used in emails. + Subject *string `json:"subject,omitempty" cborgen:"subject,omitempty"` + // updatedBy: DID of the user who is updating the template. + UpdatedBy *string `json:"updatedBy,omitempty" cborgen:"updatedBy,omitempty"` +} + +// CommunicationUpdateTemplate calls the XRPC method "tools.ozone.communication.updateTemplate". +func CommunicationUpdateTemplate(ctx context.Context, c lexutil.LexClient, input *CommunicationUpdateTemplate_Input) (*CommunicationDefs_TemplateView, error) { + var out CommunicationDefs_TemplateView + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.communication.updateTemplate", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/hostinggetAccountHistory.go b/api/ozone/hostinggetAccountHistory.go new file mode 100644 index 000000000..9180eeb2b --- /dev/null +++ b/api/ozone/hostinggetAccountHistory.go @@ -0,0 +1,137 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.hosting.getAccountHistory + +package ozone + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// HostingGetAccountHistory_AccountCreated is a "accountCreated" in the tools.ozone.hosting.getAccountHistory schema. +type HostingGetAccountHistory_AccountCreated struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.hosting.getAccountHistory#accountCreated"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + Handle *string `json:"handle,omitempty" cborgen:"handle,omitempty"` +} + +// HostingGetAccountHistory_EmailConfirmed is a "emailConfirmed" in the tools.ozone.hosting.getAccountHistory schema. +type HostingGetAccountHistory_EmailConfirmed struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.hosting.getAccountHistory#emailConfirmed"` + Email string `json:"email" cborgen:"email"` +} + +// HostingGetAccountHistory_EmailUpdated is a "emailUpdated" in the tools.ozone.hosting.getAccountHistory schema. +type HostingGetAccountHistory_EmailUpdated struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.hosting.getAccountHistory#emailUpdated"` + Email string `json:"email" cborgen:"email"` +} + +// HostingGetAccountHistory_Event is a "event" in the tools.ozone.hosting.getAccountHistory schema. +type HostingGetAccountHistory_Event struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + Details *HostingGetAccountHistory_Event_Details `json:"details" cborgen:"details"` +} + +type HostingGetAccountHistory_Event_Details struct { + HostingGetAccountHistory_AccountCreated *HostingGetAccountHistory_AccountCreated + HostingGetAccountHistory_EmailUpdated *HostingGetAccountHistory_EmailUpdated + HostingGetAccountHistory_EmailConfirmed *HostingGetAccountHistory_EmailConfirmed + HostingGetAccountHistory_PasswordUpdated *HostingGetAccountHistory_PasswordUpdated + HostingGetAccountHistory_HandleUpdated *HostingGetAccountHistory_HandleUpdated +} + +func (t *HostingGetAccountHistory_Event_Details) MarshalJSON() ([]byte, error) { + if t.HostingGetAccountHistory_AccountCreated != nil { + t.HostingGetAccountHistory_AccountCreated.LexiconTypeID = "tools.ozone.hosting.getAccountHistory#accountCreated" + return json.Marshal(t.HostingGetAccountHistory_AccountCreated) + } + if t.HostingGetAccountHistory_EmailUpdated != nil { + t.HostingGetAccountHistory_EmailUpdated.LexiconTypeID = "tools.ozone.hosting.getAccountHistory#emailUpdated" + return json.Marshal(t.HostingGetAccountHistory_EmailUpdated) + } + if t.HostingGetAccountHistory_EmailConfirmed != nil { + t.HostingGetAccountHistory_EmailConfirmed.LexiconTypeID = "tools.ozone.hosting.getAccountHistory#emailConfirmed" + return json.Marshal(t.HostingGetAccountHistory_EmailConfirmed) + } + if t.HostingGetAccountHistory_PasswordUpdated != nil { + t.HostingGetAccountHistory_PasswordUpdated.LexiconTypeID = "tools.ozone.hosting.getAccountHistory#passwordUpdated" + return json.Marshal(t.HostingGetAccountHistory_PasswordUpdated) + } + if t.HostingGetAccountHistory_HandleUpdated != nil { + t.HostingGetAccountHistory_HandleUpdated.LexiconTypeID = "tools.ozone.hosting.getAccountHistory#handleUpdated" + return json.Marshal(t.HostingGetAccountHistory_HandleUpdated) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *HostingGetAccountHistory_Event_Details) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.hosting.getAccountHistory#accountCreated": + t.HostingGetAccountHistory_AccountCreated = new(HostingGetAccountHistory_AccountCreated) + return json.Unmarshal(b, t.HostingGetAccountHistory_AccountCreated) + case "tools.ozone.hosting.getAccountHistory#emailUpdated": + t.HostingGetAccountHistory_EmailUpdated = new(HostingGetAccountHistory_EmailUpdated) + return json.Unmarshal(b, t.HostingGetAccountHistory_EmailUpdated) + case "tools.ozone.hosting.getAccountHistory#emailConfirmed": + t.HostingGetAccountHistory_EmailConfirmed = new(HostingGetAccountHistory_EmailConfirmed) + return json.Unmarshal(b, t.HostingGetAccountHistory_EmailConfirmed) + case "tools.ozone.hosting.getAccountHistory#passwordUpdated": + t.HostingGetAccountHistory_PasswordUpdated = new(HostingGetAccountHistory_PasswordUpdated) + return json.Unmarshal(b, t.HostingGetAccountHistory_PasswordUpdated) + case "tools.ozone.hosting.getAccountHistory#handleUpdated": + t.HostingGetAccountHistory_HandleUpdated = new(HostingGetAccountHistory_HandleUpdated) + return json.Unmarshal(b, t.HostingGetAccountHistory_HandleUpdated) + default: + return nil + } +} + +// HostingGetAccountHistory_HandleUpdated is a "handleUpdated" in the tools.ozone.hosting.getAccountHistory schema. +type HostingGetAccountHistory_HandleUpdated struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.hosting.getAccountHistory#handleUpdated"` + Handle string `json:"handle" cborgen:"handle"` +} + +// HostingGetAccountHistory_Output is the output of a tools.ozone.hosting.getAccountHistory call. +type HostingGetAccountHistory_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Events []*HostingGetAccountHistory_Event `json:"events" cborgen:"events"` +} + +// HostingGetAccountHistory_PasswordUpdated is a "passwordUpdated" in the tools.ozone.hosting.getAccountHistory schema. +type HostingGetAccountHistory_PasswordUpdated struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.hosting.getAccountHistory#passwordUpdated"` +} + +// HostingGetAccountHistory calls the XRPC method "tools.ozone.hosting.getAccountHistory". +func HostingGetAccountHistory(ctx context.Context, c lexutil.LexClient, cursor string, did string, events []string, limit int64) (*HostingGetAccountHistory_Output, error) { + var out HostingGetAccountHistory_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + params["did"] = did + if len(events) != 0 { + params["events"] = events + } + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.hosting.getAccountHistory", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationcancelScheduledActions.go b/api/ozone/moderationcancelScheduledActions.go new file mode 100644 index 000000000..fdc041537 --- /dev/null +++ b/api/ozone/moderationcancelScheduledActions.go @@ -0,0 +1,44 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.cancelScheduledActions + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationCancelScheduledActions_CancellationResults is a "cancellationResults" in the tools.ozone.moderation.cancelScheduledActions schema. +type ModerationCancelScheduledActions_CancellationResults struct { + // failed: DIDs for which cancellation failed with error details + Failed []*ModerationCancelScheduledActions_FailedCancellation `json:"failed" cborgen:"failed"` + // succeeded: DIDs for which all pending scheduled actions were successfully cancelled + Succeeded []string `json:"succeeded" cborgen:"succeeded"` +} + +// ModerationCancelScheduledActions_FailedCancellation is a "failedCancellation" in the tools.ozone.moderation.cancelScheduledActions schema. +type ModerationCancelScheduledActions_FailedCancellation struct { + Did string `json:"did" cborgen:"did"` + Error string `json:"error" cborgen:"error"` + ErrorCode *string `json:"errorCode,omitempty" cborgen:"errorCode,omitempty"` +} + +// ModerationCancelScheduledActions_Input is the input argument to a tools.ozone.moderation.cancelScheduledActions call. +type ModerationCancelScheduledActions_Input struct { + // comment: Optional comment describing the reason for cancellation + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // subjects: Array of DID subjects to cancel scheduled actions for + Subjects []string `json:"subjects" cborgen:"subjects"` +} + +// ModerationCancelScheduledActions calls the XRPC method "tools.ozone.moderation.cancelScheduledActions". +func ModerationCancelScheduledActions(ctx context.Context, c lexutil.LexClient, input *ModerationCancelScheduledActions_Input) (*ModerationCancelScheduledActions_CancellationResults, error) { + var out ModerationCancelScheduledActions_CancellationResults + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.moderation.cancelScheduledActions", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationdefs.go b/api/ozone/moderationdefs.go new file mode 100644 index 000000000..684e8354b --- /dev/null +++ b/api/ozone/moderationdefs.go @@ -0,0 +1,1250 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.defs + +package ozone + +import ( + "encoding/json" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + chatbsky "github.com/bluesky-social/indigo/api/chat" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationDefs_AccountEvent is a "accountEvent" in the tools.ozone.moderation.defs schema. +// +// Logs account status related events on a repo subject. Normally captured by automod from the firehose and emitted to ozone for historical tracking. +type ModerationDefs_AccountEvent struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#accountEvent"` + // active: Indicates that the account has a repository which can be fetched from the host that emitted this event. + Active bool `json:"active" cborgen:"active"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` + Timestamp string `json:"timestamp" cborgen:"timestamp"` +} + +// ModerationDefs_AccountHosting is a "accountHosting" in the tools.ozone.moderation.defs schema. +type ModerationDefs_AccountHosting struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#accountHosting"` + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` + DeletedAt *string `json:"deletedAt,omitempty" cborgen:"deletedAt,omitempty"` + ReactivatedAt *string `json:"reactivatedAt,omitempty" cborgen:"reactivatedAt,omitempty"` + Status string `json:"status" cborgen:"status"` + UpdatedAt *string `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"` +} + +// ModerationDefs_AccountStats is a "accountStats" in the tools.ozone.moderation.defs schema. +// +// Statistics about a particular account subject +type ModerationDefs_AccountStats struct { + // appealCount: Total number of appeals against a moderation action on the account + AppealCount *int64 `json:"appealCount,omitempty" cborgen:"appealCount,omitempty"` + // escalateCount: Number of times the account was escalated + EscalateCount *int64 `json:"escalateCount,omitempty" cborgen:"escalateCount,omitempty"` + // reportCount: Total number of reports on the account + ReportCount *int64 `json:"reportCount,omitempty" cborgen:"reportCount,omitempty"` + // suspendCount: Number of times the account was suspended + SuspendCount *int64 `json:"suspendCount,omitempty" cborgen:"suspendCount,omitempty"` + // takedownCount: Number of times the account was taken down + TakedownCount *int64 `json:"takedownCount,omitempty" cborgen:"takedownCount,omitempty"` +} + +// ModerationDefs_AccountStrike is a "accountStrike" in the tools.ozone.moderation.defs schema. +// +// Strike information for an account +type ModerationDefs_AccountStrike struct { + // activeStrikeCount: Current number of active strikes (excluding expired strikes) + ActiveStrikeCount *int64 `json:"activeStrikeCount,omitempty" cborgen:"activeStrikeCount,omitempty"` + // firstStrikeAt: Timestamp of the first strike received + FirstStrikeAt *string `json:"firstStrikeAt,omitempty" cborgen:"firstStrikeAt,omitempty"` + // lastStrikeAt: Timestamp of the most recent strike received + LastStrikeAt *string `json:"lastStrikeAt,omitempty" cborgen:"lastStrikeAt,omitempty"` + // totalStrikeCount: Total number of strikes ever received (including expired strikes) + TotalStrikeCount *int64 `json:"totalStrikeCount,omitempty" cborgen:"totalStrikeCount,omitempty"` +} + +// ModerationDefs_AgeAssuranceEvent is a "ageAssuranceEvent" in the tools.ozone.moderation.defs schema. +// +// Age assurance info coming directly from users. Only works on DID subjects. +type ModerationDefs_AgeAssuranceEvent struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#ageAssuranceEvent"` + Access *string `json:"access,omitempty" cborgen:"access,omitempty"` + // attemptId: The unique identifier for this instance of the age assurance flow, in UUID format. + AttemptId string `json:"attemptId" cborgen:"attemptId"` + // completeIp: The IP address used when completing the AA flow. + CompleteIp *string `json:"completeIp,omitempty" cborgen:"completeIp,omitempty"` + // completeUa: The user agent used when completing the AA flow. + CompleteUa *string `json:"completeUa,omitempty" cborgen:"completeUa,omitempty"` + // countryCode: The ISO 3166-1 alpha-2 country code provided when beginning the Age Assurance flow. + CountryCode *string `json:"countryCode,omitempty" cborgen:"countryCode,omitempty"` + // createdAt: The date and time of this write operation. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // initIp: The IP address used when initiating the AA flow. + InitIp *string `json:"initIp,omitempty" cborgen:"initIp,omitempty"` + // initUa: The user agent used when initiating the AA flow. + InitUa *string `json:"initUa,omitempty" cborgen:"initUa,omitempty"` + // regionCode: The ISO 3166-2 region code provided when beginning the Age Assurance flow. + RegionCode *string `json:"regionCode,omitempty" cborgen:"regionCode,omitempty"` + // status: The status of the Age Assurance process. + Status string `json:"status" cborgen:"status"` +} + +// ModerationDefs_AgeAssuranceOverrideEvent is a "ageAssuranceOverrideEvent" in the tools.ozone.moderation.defs schema. +// +// Age assurance status override by moderators. Only works on DID subjects. +type ModerationDefs_AgeAssuranceOverrideEvent struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#ageAssuranceOverrideEvent"` + Access *string `json:"access,omitempty" cborgen:"access,omitempty"` + // comment: Comment describing the reason for the override. + Comment string `json:"comment" cborgen:"comment"` + // status: The status to be set for the user decided by a moderator, overriding whatever value the user had previously. Use reset to default to original state. + Status string `json:"status" cborgen:"status"` +} + +// ModerationDefs_BlobView is a "blobView" in the tools.ozone.moderation.defs schema. +type ModerationDefs_BlobView struct { + Cid string `json:"cid" cborgen:"cid"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Details *ModerationDefs_BlobView_Details `json:"details,omitempty" cborgen:"details,omitempty"` + MimeType string `json:"mimeType" cborgen:"mimeType"` + Moderation *ModerationDefs_Moderation `json:"moderation,omitempty" cborgen:"moderation,omitempty"` + Size int64 `json:"size" cborgen:"size"` +} + +type ModerationDefs_BlobView_Details struct { + ModerationDefs_ImageDetails *ModerationDefs_ImageDetails + ModerationDefs_VideoDetails *ModerationDefs_VideoDetails +} + +func (t *ModerationDefs_BlobView_Details) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_ImageDetails != nil { + t.ModerationDefs_ImageDetails.LexiconTypeID = "tools.ozone.moderation.defs#imageDetails" + return json.Marshal(t.ModerationDefs_ImageDetails) + } + if t.ModerationDefs_VideoDetails != nil { + t.ModerationDefs_VideoDetails.LexiconTypeID = "tools.ozone.moderation.defs#videoDetails" + return json.Marshal(t.ModerationDefs_VideoDetails) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationDefs_BlobView_Details) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#imageDetails": + t.ModerationDefs_ImageDetails = new(ModerationDefs_ImageDetails) + return json.Unmarshal(b, t.ModerationDefs_ImageDetails) + case "tools.ozone.moderation.defs#videoDetails": + t.ModerationDefs_VideoDetails = new(ModerationDefs_VideoDetails) + return json.Unmarshal(b, t.ModerationDefs_VideoDetails) + default: + return nil + } +} + +// ModerationDefs_CancelScheduledTakedownEvent is a "cancelScheduledTakedownEvent" in the tools.ozone.moderation.defs schema. +// +// Logs cancellation of a scheduled takedown action for an account. +type ModerationDefs_CancelScheduledTakedownEvent struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#cancelScheduledTakedownEvent"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` +} + +// ModerationDefs_IdentityEvent is a "identityEvent" in the tools.ozone.moderation.defs schema. +// +// Logs identity related events on a repo subject. Normally captured by automod from the firehose and emitted to ozone for historical tracking. +type ModerationDefs_IdentityEvent struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#identityEvent"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + Handle *string `json:"handle,omitempty" cborgen:"handle,omitempty"` + PdsHost *string `json:"pdsHost,omitempty" cborgen:"pdsHost,omitempty"` + Timestamp string `json:"timestamp" cborgen:"timestamp"` + Tombstone *bool `json:"tombstone,omitempty" cborgen:"tombstone,omitempty"` +} + +// ModerationDefs_ImageDetails is a "imageDetails" in the tools.ozone.moderation.defs schema. +type ModerationDefs_ImageDetails struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#imageDetails"` + Height int64 `json:"height" cborgen:"height"` + Width int64 `json:"width" cborgen:"width"` +} + +// ModerationDefs_ModEventAcknowledge is a "modEventAcknowledge" in the tools.ozone.moderation.defs schema. +type ModerationDefs_ModEventAcknowledge struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventAcknowledge"` + // acknowledgeAccountSubjects: If true, all other reports on content authored by this account will be resolved (acknowledged). + AcknowledgeAccountSubjects *bool `json:"acknowledgeAccountSubjects,omitempty" cborgen:"acknowledgeAccountSubjects,omitempty"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` +} + +// ModerationDefs_ModEventComment is a "modEventComment" in the tools.ozone.moderation.defs schema. +// +// Add a comment to a subject. An empty comment will clear any previously set sticky comment. +type ModerationDefs_ModEventComment struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventComment"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // sticky: Make the comment persistent on the subject + Sticky *bool `json:"sticky,omitempty" cborgen:"sticky,omitempty"` +} + +// ModerationDefs_ModEventDivert is a "modEventDivert" in the tools.ozone.moderation.defs schema. +// +// Divert a record's blobs to a 3rd party service for further scanning/tagging +type ModerationDefs_ModEventDivert struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventDivert"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` +} + +// ModerationDefs_ModEventEmail is a "modEventEmail" in the tools.ozone.moderation.defs schema. +// +// Keep a log of outgoing email to a user +type ModerationDefs_ModEventEmail struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventEmail"` + // comment: Additional comment about the outgoing comm. + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // content: The content of the email sent to the user. + Content *string `json:"content,omitempty" cborgen:"content,omitempty"` + // isDelivered: Indicates whether the email was successfully delivered to the user's inbox. + IsDelivered *bool `json:"isDelivered,omitempty" cborgen:"isDelivered,omitempty"` + // policies: Names/Keywords of the policies that necessitated the email. + Policies []string `json:"policies,omitempty" cborgen:"policies,omitempty"` + // severityLevel: Severity level of the violation. Normally 'sev-1' that adds strike on repeat offense + SeverityLevel *string `json:"severityLevel,omitempty" cborgen:"severityLevel,omitempty"` + // strikeCount: Number of strikes to assign to the user for this violation. Normally 0 as an indicator of a warning and only added as a strike on a repeat offense. + StrikeCount *int64 `json:"strikeCount,omitempty" cborgen:"strikeCount,omitempty"` + // strikeExpiresAt: When the strike should expire. If not provided, the strike never expires. + StrikeExpiresAt *string `json:"strikeExpiresAt,omitempty" cborgen:"strikeExpiresAt,omitempty"` + // subjectLine: The subject line of the email sent to the user. + SubjectLine string `json:"subjectLine" cborgen:"subjectLine"` +} + +// ModerationDefs_ModEventEscalate is a "modEventEscalate" in the tools.ozone.moderation.defs schema. +type ModerationDefs_ModEventEscalate struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventEscalate"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` +} + +// ModerationDefs_ModEventLabel is a "modEventLabel" in the tools.ozone.moderation.defs schema. +// +// Apply/Negate labels on a subject +type ModerationDefs_ModEventLabel struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventLabel"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + CreateLabelVals []string `json:"createLabelVals" cborgen:"createLabelVals"` + // durationInHours: Indicates how long the label will remain on the subject. Only applies on labels that are being added. + DurationInHours *int64 `json:"durationInHours,omitempty" cborgen:"durationInHours,omitempty"` + NegateLabelVals []string `json:"negateLabelVals" cborgen:"negateLabelVals"` +} + +// ModerationDefs_ModEventMute is a "modEventMute" in the tools.ozone.moderation.defs schema. +// +// Mute incoming reports on a subject +type ModerationDefs_ModEventMute struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventMute"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // durationInHours: Indicates how long the subject should remain muted. + DurationInHours int64 `json:"durationInHours" cborgen:"durationInHours"` +} + +// ModerationDefs_ModEventMuteReporter is a "modEventMuteReporter" in the tools.ozone.moderation.defs schema. +// +// Mute incoming reports from an account +type ModerationDefs_ModEventMuteReporter struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventMuteReporter"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // durationInHours: Indicates how long the account should remain muted. Falsy value here means a permanent mute. + DurationInHours *int64 `json:"durationInHours,omitempty" cborgen:"durationInHours,omitempty"` +} + +// ModerationDefs_ModEventPriorityScore is a "modEventPriorityScore" in the tools.ozone.moderation.defs schema. +// +// Set priority score of the subject. Higher score means higher priority. +type ModerationDefs_ModEventPriorityScore struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventPriorityScore"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + Score int64 `json:"score" cborgen:"score"` +} + +// ModerationDefs_ModEventReport is a "modEventReport" in the tools.ozone.moderation.defs schema. +// +// Report a subject +type ModerationDefs_ModEventReport struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventReport"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // isReporterMuted: Set to true if the reporter was muted from reporting at the time of the event. These reports won't impact the reviewState of the subject. + IsReporterMuted *bool `json:"isReporterMuted,omitempty" cborgen:"isReporterMuted,omitempty"` + ReportType *string `json:"reportType" cborgen:"reportType"` +} + +// ModerationDefs_ModEventResolveAppeal is a "modEventResolveAppeal" in the tools.ozone.moderation.defs schema. +// +// Resolve appeal on a subject +type ModerationDefs_ModEventResolveAppeal struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventResolveAppeal"` + // comment: Describe resolution. + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` +} + +// ModerationDefs_ModEventReverseTakedown is a "modEventReverseTakedown" in the tools.ozone.moderation.defs schema. +// +// Revert take down action on a subject +type ModerationDefs_ModEventReverseTakedown struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventReverseTakedown"` + // comment: Describe reasoning behind the reversal. + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // policies: Names/Keywords of the policy infraction for which takedown is being reversed. + Policies []string `json:"policies,omitempty" cborgen:"policies,omitempty"` + // severityLevel: Severity level of the violation. Usually set from the last policy infraction's severity. + SeverityLevel *string `json:"severityLevel,omitempty" cborgen:"severityLevel,omitempty"` + // strikeCount: Number of strikes to subtract from the user's strike count. Usually set from the last policy infraction's severity. + StrikeCount *int64 `json:"strikeCount,omitempty" cborgen:"strikeCount,omitempty"` +} + +// ModerationDefs_ModEventTag is a "modEventTag" in the tools.ozone.moderation.defs schema. +// +// Add/Remove a tag on a subject +type ModerationDefs_ModEventTag struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventTag"` + // add: Tags to be added to the subject. If already exists, won't be duplicated. + Add []string `json:"add" cborgen:"add"` + // comment: Additional comment about added/removed tags. + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // remove: Tags to be removed to the subject. Ignores a tag If it doesn't exist, won't be duplicated. + Remove []string `json:"remove" cborgen:"remove"` +} + +// ModerationDefs_ModEventTakedown is a "modEventTakedown" in the tools.ozone.moderation.defs schema. +// +// Take down a subject permanently or temporarily +type ModerationDefs_ModEventTakedown struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventTakedown"` + // acknowledgeAccountSubjects: If true, all other reports on content authored by this account will be resolved (acknowledged). + AcknowledgeAccountSubjects *bool `json:"acknowledgeAccountSubjects,omitempty" cborgen:"acknowledgeAccountSubjects,omitempty"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // durationInHours: Indicates how long the takedown should be in effect before automatically expiring. + DurationInHours *int64 `json:"durationInHours,omitempty" cborgen:"durationInHours,omitempty"` + // policies: Names/Keywords of the policies that drove the decision. + Policies []string `json:"policies,omitempty" cborgen:"policies,omitempty"` + // severityLevel: Severity level of the violation (e.g., 'sev-0', 'sev-1', 'sev-2', etc.). + SeverityLevel *string `json:"severityLevel,omitempty" cborgen:"severityLevel,omitempty"` + // strikeCount: Number of strikes to assign to the user for this violation. + StrikeCount *int64 `json:"strikeCount,omitempty" cborgen:"strikeCount,omitempty"` + // strikeExpiresAt: When the strike should expire. If not provided, the strike never expires. + StrikeExpiresAt *string `json:"strikeExpiresAt,omitempty" cborgen:"strikeExpiresAt,omitempty"` + // targetServices: List of services where the takedown should be applied. If empty or not provided, takedown is applied on all configured services. + TargetServices []string `json:"targetServices,omitempty" cborgen:"targetServices,omitempty"` +} + +// ModerationDefs_ModEventUnmute is a "modEventUnmute" in the tools.ozone.moderation.defs schema. +// +// Unmute action on a subject +type ModerationDefs_ModEventUnmute struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventUnmute"` + // comment: Describe reasoning behind the reversal. + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` +} + +// ModerationDefs_ModEventUnmuteReporter is a "modEventUnmuteReporter" in the tools.ozone.moderation.defs schema. +// +// Unmute incoming reports from an account +type ModerationDefs_ModEventUnmuteReporter struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#modEventUnmuteReporter"` + // comment: Describe reasoning behind the reversal. + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` +} + +// ModerationDefs_ModEventView is a "modEventView" in the tools.ozone.moderation.defs schema. +type ModerationDefs_ModEventView struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + CreatorHandle *string `json:"creatorHandle,omitempty" cborgen:"creatorHandle,omitempty"` + Event *ModerationDefs_ModEventView_Event `json:"event" cborgen:"event"` + Id int64 `json:"id" cborgen:"id"` + ModTool *ModerationDefs_ModTool `json:"modTool,omitempty" cborgen:"modTool,omitempty"` + Subject *ModerationDefs_ModEventView_Subject `json:"subject" cborgen:"subject"` + SubjectBlobCids []string `json:"subjectBlobCids" cborgen:"subjectBlobCids"` + SubjectHandle *string `json:"subjectHandle,omitempty" cborgen:"subjectHandle,omitempty"` +} + +// ModerationDefs_ModEventViewDetail is a "modEventViewDetail" in the tools.ozone.moderation.defs schema. +type ModerationDefs_ModEventViewDetail struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + Event *ModerationDefs_ModEventViewDetail_Event `json:"event" cborgen:"event"` + Id int64 `json:"id" cborgen:"id"` + ModTool *ModerationDefs_ModTool `json:"modTool,omitempty" cborgen:"modTool,omitempty"` + Subject *ModerationDefs_ModEventViewDetail_Subject `json:"subject" cborgen:"subject"` + SubjectBlobs []*ModerationDefs_BlobView `json:"subjectBlobs" cborgen:"subjectBlobs"` +} + +type ModerationDefs_ModEventViewDetail_Event struct { + ModerationDefs_ModEventTakedown *ModerationDefs_ModEventTakedown + ModerationDefs_ModEventReverseTakedown *ModerationDefs_ModEventReverseTakedown + ModerationDefs_ModEventComment *ModerationDefs_ModEventComment + ModerationDefs_ModEventReport *ModerationDefs_ModEventReport + ModerationDefs_ModEventLabel *ModerationDefs_ModEventLabel + ModerationDefs_ModEventAcknowledge *ModerationDefs_ModEventAcknowledge + ModerationDefs_ModEventEscalate *ModerationDefs_ModEventEscalate + ModerationDefs_ModEventMute *ModerationDefs_ModEventMute + ModerationDefs_ModEventUnmute *ModerationDefs_ModEventUnmute + ModerationDefs_ModEventMuteReporter *ModerationDefs_ModEventMuteReporter + ModerationDefs_ModEventUnmuteReporter *ModerationDefs_ModEventUnmuteReporter + ModerationDefs_ModEventEmail *ModerationDefs_ModEventEmail + ModerationDefs_ModEventResolveAppeal *ModerationDefs_ModEventResolveAppeal + ModerationDefs_ModEventDivert *ModerationDefs_ModEventDivert + ModerationDefs_ModEventTag *ModerationDefs_ModEventTag + ModerationDefs_AccountEvent *ModerationDefs_AccountEvent + ModerationDefs_IdentityEvent *ModerationDefs_IdentityEvent + ModerationDefs_RecordEvent *ModerationDefs_RecordEvent + ModerationDefs_ModEventPriorityScore *ModerationDefs_ModEventPriorityScore + ModerationDefs_AgeAssuranceEvent *ModerationDefs_AgeAssuranceEvent + ModerationDefs_AgeAssuranceOverrideEvent *ModerationDefs_AgeAssuranceOverrideEvent + ModerationDefs_RevokeAccountCredentialsEvent *ModerationDefs_RevokeAccountCredentialsEvent + ModerationDefs_ScheduleTakedownEvent *ModerationDefs_ScheduleTakedownEvent + ModerationDefs_CancelScheduledTakedownEvent *ModerationDefs_CancelScheduledTakedownEvent +} + +func (t *ModerationDefs_ModEventViewDetail_Event) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_ModEventTakedown != nil { + t.ModerationDefs_ModEventTakedown.LexiconTypeID = "tools.ozone.moderation.defs#modEventTakedown" + return json.Marshal(t.ModerationDefs_ModEventTakedown) + } + if t.ModerationDefs_ModEventReverseTakedown != nil { + t.ModerationDefs_ModEventReverseTakedown.LexiconTypeID = "tools.ozone.moderation.defs#modEventReverseTakedown" + return json.Marshal(t.ModerationDefs_ModEventReverseTakedown) + } + if t.ModerationDefs_ModEventComment != nil { + t.ModerationDefs_ModEventComment.LexiconTypeID = "tools.ozone.moderation.defs#modEventComment" + return json.Marshal(t.ModerationDefs_ModEventComment) + } + if t.ModerationDefs_ModEventReport != nil { + t.ModerationDefs_ModEventReport.LexiconTypeID = "tools.ozone.moderation.defs#modEventReport" + return json.Marshal(t.ModerationDefs_ModEventReport) + } + if t.ModerationDefs_ModEventLabel != nil { + t.ModerationDefs_ModEventLabel.LexiconTypeID = "tools.ozone.moderation.defs#modEventLabel" + return json.Marshal(t.ModerationDefs_ModEventLabel) + } + if t.ModerationDefs_ModEventAcknowledge != nil { + t.ModerationDefs_ModEventAcknowledge.LexiconTypeID = "tools.ozone.moderation.defs#modEventAcknowledge" + return json.Marshal(t.ModerationDefs_ModEventAcknowledge) + } + if t.ModerationDefs_ModEventEscalate != nil { + t.ModerationDefs_ModEventEscalate.LexiconTypeID = "tools.ozone.moderation.defs#modEventEscalate" + return json.Marshal(t.ModerationDefs_ModEventEscalate) + } + if t.ModerationDefs_ModEventMute != nil { + t.ModerationDefs_ModEventMute.LexiconTypeID = "tools.ozone.moderation.defs#modEventMute" + return json.Marshal(t.ModerationDefs_ModEventMute) + } + if t.ModerationDefs_ModEventUnmute != nil { + t.ModerationDefs_ModEventUnmute.LexiconTypeID = "tools.ozone.moderation.defs#modEventUnmute" + return json.Marshal(t.ModerationDefs_ModEventUnmute) + } + if t.ModerationDefs_ModEventMuteReporter != nil { + t.ModerationDefs_ModEventMuteReporter.LexiconTypeID = "tools.ozone.moderation.defs#modEventMuteReporter" + return json.Marshal(t.ModerationDefs_ModEventMuteReporter) + } + if t.ModerationDefs_ModEventUnmuteReporter != nil { + t.ModerationDefs_ModEventUnmuteReporter.LexiconTypeID = "tools.ozone.moderation.defs#modEventUnmuteReporter" + return json.Marshal(t.ModerationDefs_ModEventUnmuteReporter) + } + if t.ModerationDefs_ModEventEmail != nil { + t.ModerationDefs_ModEventEmail.LexiconTypeID = "tools.ozone.moderation.defs#modEventEmail" + return json.Marshal(t.ModerationDefs_ModEventEmail) + } + if t.ModerationDefs_ModEventResolveAppeal != nil { + t.ModerationDefs_ModEventResolveAppeal.LexiconTypeID = "tools.ozone.moderation.defs#modEventResolveAppeal" + return json.Marshal(t.ModerationDefs_ModEventResolveAppeal) + } + if t.ModerationDefs_ModEventDivert != nil { + t.ModerationDefs_ModEventDivert.LexiconTypeID = "tools.ozone.moderation.defs#modEventDivert" + return json.Marshal(t.ModerationDefs_ModEventDivert) + } + if t.ModerationDefs_ModEventTag != nil { + t.ModerationDefs_ModEventTag.LexiconTypeID = "tools.ozone.moderation.defs#modEventTag" + return json.Marshal(t.ModerationDefs_ModEventTag) + } + if t.ModerationDefs_AccountEvent != nil { + t.ModerationDefs_AccountEvent.LexiconTypeID = "tools.ozone.moderation.defs#accountEvent" + return json.Marshal(t.ModerationDefs_AccountEvent) + } + if t.ModerationDefs_IdentityEvent != nil { + t.ModerationDefs_IdentityEvent.LexiconTypeID = "tools.ozone.moderation.defs#identityEvent" + return json.Marshal(t.ModerationDefs_IdentityEvent) + } + if t.ModerationDefs_RecordEvent != nil { + t.ModerationDefs_RecordEvent.LexiconTypeID = "tools.ozone.moderation.defs#recordEvent" + return json.Marshal(t.ModerationDefs_RecordEvent) + } + if t.ModerationDefs_ModEventPriorityScore != nil { + t.ModerationDefs_ModEventPriorityScore.LexiconTypeID = "tools.ozone.moderation.defs#modEventPriorityScore" + return json.Marshal(t.ModerationDefs_ModEventPriorityScore) + } + if t.ModerationDefs_AgeAssuranceEvent != nil { + t.ModerationDefs_AgeAssuranceEvent.LexiconTypeID = "tools.ozone.moderation.defs#ageAssuranceEvent" + return json.Marshal(t.ModerationDefs_AgeAssuranceEvent) + } + if t.ModerationDefs_AgeAssuranceOverrideEvent != nil { + t.ModerationDefs_AgeAssuranceOverrideEvent.LexiconTypeID = "tools.ozone.moderation.defs#ageAssuranceOverrideEvent" + return json.Marshal(t.ModerationDefs_AgeAssuranceOverrideEvent) + } + if t.ModerationDefs_RevokeAccountCredentialsEvent != nil { + t.ModerationDefs_RevokeAccountCredentialsEvent.LexiconTypeID = "tools.ozone.moderation.defs#revokeAccountCredentialsEvent" + return json.Marshal(t.ModerationDefs_RevokeAccountCredentialsEvent) + } + if t.ModerationDefs_ScheduleTakedownEvent != nil { + t.ModerationDefs_ScheduleTakedownEvent.LexiconTypeID = "tools.ozone.moderation.defs#scheduleTakedownEvent" + return json.Marshal(t.ModerationDefs_ScheduleTakedownEvent) + } + if t.ModerationDefs_CancelScheduledTakedownEvent != nil { + t.ModerationDefs_CancelScheduledTakedownEvent.LexiconTypeID = "tools.ozone.moderation.defs#cancelScheduledTakedownEvent" + return json.Marshal(t.ModerationDefs_CancelScheduledTakedownEvent) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationDefs_ModEventViewDetail_Event) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#modEventTakedown": + t.ModerationDefs_ModEventTakedown = new(ModerationDefs_ModEventTakedown) + return json.Unmarshal(b, t.ModerationDefs_ModEventTakedown) + case "tools.ozone.moderation.defs#modEventReverseTakedown": + t.ModerationDefs_ModEventReverseTakedown = new(ModerationDefs_ModEventReverseTakedown) + return json.Unmarshal(b, t.ModerationDefs_ModEventReverseTakedown) + case "tools.ozone.moderation.defs#modEventComment": + t.ModerationDefs_ModEventComment = new(ModerationDefs_ModEventComment) + return json.Unmarshal(b, t.ModerationDefs_ModEventComment) + case "tools.ozone.moderation.defs#modEventReport": + t.ModerationDefs_ModEventReport = new(ModerationDefs_ModEventReport) + return json.Unmarshal(b, t.ModerationDefs_ModEventReport) + case "tools.ozone.moderation.defs#modEventLabel": + t.ModerationDefs_ModEventLabel = new(ModerationDefs_ModEventLabel) + return json.Unmarshal(b, t.ModerationDefs_ModEventLabel) + case "tools.ozone.moderation.defs#modEventAcknowledge": + t.ModerationDefs_ModEventAcknowledge = new(ModerationDefs_ModEventAcknowledge) + return json.Unmarshal(b, t.ModerationDefs_ModEventAcknowledge) + case "tools.ozone.moderation.defs#modEventEscalate": + t.ModerationDefs_ModEventEscalate = new(ModerationDefs_ModEventEscalate) + return json.Unmarshal(b, t.ModerationDefs_ModEventEscalate) + case "tools.ozone.moderation.defs#modEventMute": + t.ModerationDefs_ModEventMute = new(ModerationDefs_ModEventMute) + return json.Unmarshal(b, t.ModerationDefs_ModEventMute) + case "tools.ozone.moderation.defs#modEventUnmute": + t.ModerationDefs_ModEventUnmute = new(ModerationDefs_ModEventUnmute) + return json.Unmarshal(b, t.ModerationDefs_ModEventUnmute) + case "tools.ozone.moderation.defs#modEventMuteReporter": + t.ModerationDefs_ModEventMuteReporter = new(ModerationDefs_ModEventMuteReporter) + return json.Unmarshal(b, t.ModerationDefs_ModEventMuteReporter) + case "tools.ozone.moderation.defs#modEventUnmuteReporter": + t.ModerationDefs_ModEventUnmuteReporter = new(ModerationDefs_ModEventUnmuteReporter) + return json.Unmarshal(b, t.ModerationDefs_ModEventUnmuteReporter) + case "tools.ozone.moderation.defs#modEventEmail": + t.ModerationDefs_ModEventEmail = new(ModerationDefs_ModEventEmail) + return json.Unmarshal(b, t.ModerationDefs_ModEventEmail) + case "tools.ozone.moderation.defs#modEventResolveAppeal": + t.ModerationDefs_ModEventResolveAppeal = new(ModerationDefs_ModEventResolveAppeal) + return json.Unmarshal(b, t.ModerationDefs_ModEventResolveAppeal) + case "tools.ozone.moderation.defs#modEventDivert": + t.ModerationDefs_ModEventDivert = new(ModerationDefs_ModEventDivert) + return json.Unmarshal(b, t.ModerationDefs_ModEventDivert) + case "tools.ozone.moderation.defs#modEventTag": + t.ModerationDefs_ModEventTag = new(ModerationDefs_ModEventTag) + return json.Unmarshal(b, t.ModerationDefs_ModEventTag) + case "tools.ozone.moderation.defs#accountEvent": + t.ModerationDefs_AccountEvent = new(ModerationDefs_AccountEvent) + return json.Unmarshal(b, t.ModerationDefs_AccountEvent) + case "tools.ozone.moderation.defs#identityEvent": + t.ModerationDefs_IdentityEvent = new(ModerationDefs_IdentityEvent) + return json.Unmarshal(b, t.ModerationDefs_IdentityEvent) + case "tools.ozone.moderation.defs#recordEvent": + t.ModerationDefs_RecordEvent = new(ModerationDefs_RecordEvent) + return json.Unmarshal(b, t.ModerationDefs_RecordEvent) + case "tools.ozone.moderation.defs#modEventPriorityScore": + t.ModerationDefs_ModEventPriorityScore = new(ModerationDefs_ModEventPriorityScore) + return json.Unmarshal(b, t.ModerationDefs_ModEventPriorityScore) + case "tools.ozone.moderation.defs#ageAssuranceEvent": + t.ModerationDefs_AgeAssuranceEvent = new(ModerationDefs_AgeAssuranceEvent) + return json.Unmarshal(b, t.ModerationDefs_AgeAssuranceEvent) + case "tools.ozone.moderation.defs#ageAssuranceOverrideEvent": + t.ModerationDefs_AgeAssuranceOverrideEvent = new(ModerationDefs_AgeAssuranceOverrideEvent) + return json.Unmarshal(b, t.ModerationDefs_AgeAssuranceOverrideEvent) + case "tools.ozone.moderation.defs#revokeAccountCredentialsEvent": + t.ModerationDefs_RevokeAccountCredentialsEvent = new(ModerationDefs_RevokeAccountCredentialsEvent) + return json.Unmarshal(b, t.ModerationDefs_RevokeAccountCredentialsEvent) + case "tools.ozone.moderation.defs#scheduleTakedownEvent": + t.ModerationDefs_ScheduleTakedownEvent = new(ModerationDefs_ScheduleTakedownEvent) + return json.Unmarshal(b, t.ModerationDefs_ScheduleTakedownEvent) + case "tools.ozone.moderation.defs#cancelScheduledTakedownEvent": + t.ModerationDefs_CancelScheduledTakedownEvent = new(ModerationDefs_CancelScheduledTakedownEvent) + return json.Unmarshal(b, t.ModerationDefs_CancelScheduledTakedownEvent) + default: + return nil + } +} + +type ModerationDefs_ModEventViewDetail_Subject struct { + ModerationDefs_RepoView *ModerationDefs_RepoView + ModerationDefs_RepoViewNotFound *ModerationDefs_RepoViewNotFound + ModerationDefs_RecordView *ModerationDefs_RecordView + ModerationDefs_RecordViewNotFound *ModerationDefs_RecordViewNotFound +} + +func (t *ModerationDefs_ModEventViewDetail_Subject) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_RepoView != nil { + t.ModerationDefs_RepoView.LexiconTypeID = "tools.ozone.moderation.defs#repoView" + return json.Marshal(t.ModerationDefs_RepoView) + } + if t.ModerationDefs_RepoViewNotFound != nil { + t.ModerationDefs_RepoViewNotFound.LexiconTypeID = "tools.ozone.moderation.defs#repoViewNotFound" + return json.Marshal(t.ModerationDefs_RepoViewNotFound) + } + if t.ModerationDefs_RecordView != nil { + t.ModerationDefs_RecordView.LexiconTypeID = "tools.ozone.moderation.defs#recordView" + return json.Marshal(t.ModerationDefs_RecordView) + } + if t.ModerationDefs_RecordViewNotFound != nil { + t.ModerationDefs_RecordViewNotFound.LexiconTypeID = "tools.ozone.moderation.defs#recordViewNotFound" + return json.Marshal(t.ModerationDefs_RecordViewNotFound) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationDefs_ModEventViewDetail_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#repoView": + t.ModerationDefs_RepoView = new(ModerationDefs_RepoView) + return json.Unmarshal(b, t.ModerationDefs_RepoView) + case "tools.ozone.moderation.defs#repoViewNotFound": + t.ModerationDefs_RepoViewNotFound = new(ModerationDefs_RepoViewNotFound) + return json.Unmarshal(b, t.ModerationDefs_RepoViewNotFound) + case "tools.ozone.moderation.defs#recordView": + t.ModerationDefs_RecordView = new(ModerationDefs_RecordView) + return json.Unmarshal(b, t.ModerationDefs_RecordView) + case "tools.ozone.moderation.defs#recordViewNotFound": + t.ModerationDefs_RecordViewNotFound = new(ModerationDefs_RecordViewNotFound) + return json.Unmarshal(b, t.ModerationDefs_RecordViewNotFound) + default: + return nil + } +} + +type ModerationDefs_ModEventView_Event struct { + ModerationDefs_ModEventTakedown *ModerationDefs_ModEventTakedown + ModerationDefs_ModEventReverseTakedown *ModerationDefs_ModEventReverseTakedown + ModerationDefs_ModEventComment *ModerationDefs_ModEventComment + ModerationDefs_ModEventReport *ModerationDefs_ModEventReport + ModerationDefs_ModEventLabel *ModerationDefs_ModEventLabel + ModerationDefs_ModEventAcknowledge *ModerationDefs_ModEventAcknowledge + ModerationDefs_ModEventEscalate *ModerationDefs_ModEventEscalate + ModerationDefs_ModEventMute *ModerationDefs_ModEventMute + ModerationDefs_ModEventUnmute *ModerationDefs_ModEventUnmute + ModerationDefs_ModEventMuteReporter *ModerationDefs_ModEventMuteReporter + ModerationDefs_ModEventUnmuteReporter *ModerationDefs_ModEventUnmuteReporter + ModerationDefs_ModEventEmail *ModerationDefs_ModEventEmail + ModerationDefs_ModEventResolveAppeal *ModerationDefs_ModEventResolveAppeal + ModerationDefs_ModEventDivert *ModerationDefs_ModEventDivert + ModerationDefs_ModEventTag *ModerationDefs_ModEventTag + ModerationDefs_AccountEvent *ModerationDefs_AccountEvent + ModerationDefs_IdentityEvent *ModerationDefs_IdentityEvent + ModerationDefs_RecordEvent *ModerationDefs_RecordEvent + ModerationDefs_ModEventPriorityScore *ModerationDefs_ModEventPriorityScore + ModerationDefs_AgeAssuranceEvent *ModerationDefs_AgeAssuranceEvent + ModerationDefs_AgeAssuranceOverrideEvent *ModerationDefs_AgeAssuranceOverrideEvent + ModerationDefs_RevokeAccountCredentialsEvent *ModerationDefs_RevokeAccountCredentialsEvent + ModerationDefs_ScheduleTakedownEvent *ModerationDefs_ScheduleTakedownEvent + ModerationDefs_CancelScheduledTakedownEvent *ModerationDefs_CancelScheduledTakedownEvent +} + +func (t *ModerationDefs_ModEventView_Event) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_ModEventTakedown != nil { + t.ModerationDefs_ModEventTakedown.LexiconTypeID = "tools.ozone.moderation.defs#modEventTakedown" + return json.Marshal(t.ModerationDefs_ModEventTakedown) + } + if t.ModerationDefs_ModEventReverseTakedown != nil { + t.ModerationDefs_ModEventReverseTakedown.LexiconTypeID = "tools.ozone.moderation.defs#modEventReverseTakedown" + return json.Marshal(t.ModerationDefs_ModEventReverseTakedown) + } + if t.ModerationDefs_ModEventComment != nil { + t.ModerationDefs_ModEventComment.LexiconTypeID = "tools.ozone.moderation.defs#modEventComment" + return json.Marshal(t.ModerationDefs_ModEventComment) + } + if t.ModerationDefs_ModEventReport != nil { + t.ModerationDefs_ModEventReport.LexiconTypeID = "tools.ozone.moderation.defs#modEventReport" + return json.Marshal(t.ModerationDefs_ModEventReport) + } + if t.ModerationDefs_ModEventLabel != nil { + t.ModerationDefs_ModEventLabel.LexiconTypeID = "tools.ozone.moderation.defs#modEventLabel" + return json.Marshal(t.ModerationDefs_ModEventLabel) + } + if t.ModerationDefs_ModEventAcknowledge != nil { + t.ModerationDefs_ModEventAcknowledge.LexiconTypeID = "tools.ozone.moderation.defs#modEventAcknowledge" + return json.Marshal(t.ModerationDefs_ModEventAcknowledge) + } + if t.ModerationDefs_ModEventEscalate != nil { + t.ModerationDefs_ModEventEscalate.LexiconTypeID = "tools.ozone.moderation.defs#modEventEscalate" + return json.Marshal(t.ModerationDefs_ModEventEscalate) + } + if t.ModerationDefs_ModEventMute != nil { + t.ModerationDefs_ModEventMute.LexiconTypeID = "tools.ozone.moderation.defs#modEventMute" + return json.Marshal(t.ModerationDefs_ModEventMute) + } + if t.ModerationDefs_ModEventUnmute != nil { + t.ModerationDefs_ModEventUnmute.LexiconTypeID = "tools.ozone.moderation.defs#modEventUnmute" + return json.Marshal(t.ModerationDefs_ModEventUnmute) + } + if t.ModerationDefs_ModEventMuteReporter != nil { + t.ModerationDefs_ModEventMuteReporter.LexiconTypeID = "tools.ozone.moderation.defs#modEventMuteReporter" + return json.Marshal(t.ModerationDefs_ModEventMuteReporter) + } + if t.ModerationDefs_ModEventUnmuteReporter != nil { + t.ModerationDefs_ModEventUnmuteReporter.LexiconTypeID = "tools.ozone.moderation.defs#modEventUnmuteReporter" + return json.Marshal(t.ModerationDefs_ModEventUnmuteReporter) + } + if t.ModerationDefs_ModEventEmail != nil { + t.ModerationDefs_ModEventEmail.LexiconTypeID = "tools.ozone.moderation.defs#modEventEmail" + return json.Marshal(t.ModerationDefs_ModEventEmail) + } + if t.ModerationDefs_ModEventResolveAppeal != nil { + t.ModerationDefs_ModEventResolveAppeal.LexiconTypeID = "tools.ozone.moderation.defs#modEventResolveAppeal" + return json.Marshal(t.ModerationDefs_ModEventResolveAppeal) + } + if t.ModerationDefs_ModEventDivert != nil { + t.ModerationDefs_ModEventDivert.LexiconTypeID = "tools.ozone.moderation.defs#modEventDivert" + return json.Marshal(t.ModerationDefs_ModEventDivert) + } + if t.ModerationDefs_ModEventTag != nil { + t.ModerationDefs_ModEventTag.LexiconTypeID = "tools.ozone.moderation.defs#modEventTag" + return json.Marshal(t.ModerationDefs_ModEventTag) + } + if t.ModerationDefs_AccountEvent != nil { + t.ModerationDefs_AccountEvent.LexiconTypeID = "tools.ozone.moderation.defs#accountEvent" + return json.Marshal(t.ModerationDefs_AccountEvent) + } + if t.ModerationDefs_IdentityEvent != nil { + t.ModerationDefs_IdentityEvent.LexiconTypeID = "tools.ozone.moderation.defs#identityEvent" + return json.Marshal(t.ModerationDefs_IdentityEvent) + } + if t.ModerationDefs_RecordEvent != nil { + t.ModerationDefs_RecordEvent.LexiconTypeID = "tools.ozone.moderation.defs#recordEvent" + return json.Marshal(t.ModerationDefs_RecordEvent) + } + if t.ModerationDefs_ModEventPriorityScore != nil { + t.ModerationDefs_ModEventPriorityScore.LexiconTypeID = "tools.ozone.moderation.defs#modEventPriorityScore" + return json.Marshal(t.ModerationDefs_ModEventPriorityScore) + } + if t.ModerationDefs_AgeAssuranceEvent != nil { + t.ModerationDefs_AgeAssuranceEvent.LexiconTypeID = "tools.ozone.moderation.defs#ageAssuranceEvent" + return json.Marshal(t.ModerationDefs_AgeAssuranceEvent) + } + if t.ModerationDefs_AgeAssuranceOverrideEvent != nil { + t.ModerationDefs_AgeAssuranceOverrideEvent.LexiconTypeID = "tools.ozone.moderation.defs#ageAssuranceOverrideEvent" + return json.Marshal(t.ModerationDefs_AgeAssuranceOverrideEvent) + } + if t.ModerationDefs_RevokeAccountCredentialsEvent != nil { + t.ModerationDefs_RevokeAccountCredentialsEvent.LexiconTypeID = "tools.ozone.moderation.defs#revokeAccountCredentialsEvent" + return json.Marshal(t.ModerationDefs_RevokeAccountCredentialsEvent) + } + if t.ModerationDefs_ScheduleTakedownEvent != nil { + t.ModerationDefs_ScheduleTakedownEvent.LexiconTypeID = "tools.ozone.moderation.defs#scheduleTakedownEvent" + return json.Marshal(t.ModerationDefs_ScheduleTakedownEvent) + } + if t.ModerationDefs_CancelScheduledTakedownEvent != nil { + t.ModerationDefs_CancelScheduledTakedownEvent.LexiconTypeID = "tools.ozone.moderation.defs#cancelScheduledTakedownEvent" + return json.Marshal(t.ModerationDefs_CancelScheduledTakedownEvent) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationDefs_ModEventView_Event) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#modEventTakedown": + t.ModerationDefs_ModEventTakedown = new(ModerationDefs_ModEventTakedown) + return json.Unmarshal(b, t.ModerationDefs_ModEventTakedown) + case "tools.ozone.moderation.defs#modEventReverseTakedown": + t.ModerationDefs_ModEventReverseTakedown = new(ModerationDefs_ModEventReverseTakedown) + return json.Unmarshal(b, t.ModerationDefs_ModEventReverseTakedown) + case "tools.ozone.moderation.defs#modEventComment": + t.ModerationDefs_ModEventComment = new(ModerationDefs_ModEventComment) + return json.Unmarshal(b, t.ModerationDefs_ModEventComment) + case "tools.ozone.moderation.defs#modEventReport": + t.ModerationDefs_ModEventReport = new(ModerationDefs_ModEventReport) + return json.Unmarshal(b, t.ModerationDefs_ModEventReport) + case "tools.ozone.moderation.defs#modEventLabel": + t.ModerationDefs_ModEventLabel = new(ModerationDefs_ModEventLabel) + return json.Unmarshal(b, t.ModerationDefs_ModEventLabel) + case "tools.ozone.moderation.defs#modEventAcknowledge": + t.ModerationDefs_ModEventAcknowledge = new(ModerationDefs_ModEventAcknowledge) + return json.Unmarshal(b, t.ModerationDefs_ModEventAcknowledge) + case "tools.ozone.moderation.defs#modEventEscalate": + t.ModerationDefs_ModEventEscalate = new(ModerationDefs_ModEventEscalate) + return json.Unmarshal(b, t.ModerationDefs_ModEventEscalate) + case "tools.ozone.moderation.defs#modEventMute": + t.ModerationDefs_ModEventMute = new(ModerationDefs_ModEventMute) + return json.Unmarshal(b, t.ModerationDefs_ModEventMute) + case "tools.ozone.moderation.defs#modEventUnmute": + t.ModerationDefs_ModEventUnmute = new(ModerationDefs_ModEventUnmute) + return json.Unmarshal(b, t.ModerationDefs_ModEventUnmute) + case "tools.ozone.moderation.defs#modEventMuteReporter": + t.ModerationDefs_ModEventMuteReporter = new(ModerationDefs_ModEventMuteReporter) + return json.Unmarshal(b, t.ModerationDefs_ModEventMuteReporter) + case "tools.ozone.moderation.defs#modEventUnmuteReporter": + t.ModerationDefs_ModEventUnmuteReporter = new(ModerationDefs_ModEventUnmuteReporter) + return json.Unmarshal(b, t.ModerationDefs_ModEventUnmuteReporter) + case "tools.ozone.moderation.defs#modEventEmail": + t.ModerationDefs_ModEventEmail = new(ModerationDefs_ModEventEmail) + return json.Unmarshal(b, t.ModerationDefs_ModEventEmail) + case "tools.ozone.moderation.defs#modEventResolveAppeal": + t.ModerationDefs_ModEventResolveAppeal = new(ModerationDefs_ModEventResolveAppeal) + return json.Unmarshal(b, t.ModerationDefs_ModEventResolveAppeal) + case "tools.ozone.moderation.defs#modEventDivert": + t.ModerationDefs_ModEventDivert = new(ModerationDefs_ModEventDivert) + return json.Unmarshal(b, t.ModerationDefs_ModEventDivert) + case "tools.ozone.moderation.defs#modEventTag": + t.ModerationDefs_ModEventTag = new(ModerationDefs_ModEventTag) + return json.Unmarshal(b, t.ModerationDefs_ModEventTag) + case "tools.ozone.moderation.defs#accountEvent": + t.ModerationDefs_AccountEvent = new(ModerationDefs_AccountEvent) + return json.Unmarshal(b, t.ModerationDefs_AccountEvent) + case "tools.ozone.moderation.defs#identityEvent": + t.ModerationDefs_IdentityEvent = new(ModerationDefs_IdentityEvent) + return json.Unmarshal(b, t.ModerationDefs_IdentityEvent) + case "tools.ozone.moderation.defs#recordEvent": + t.ModerationDefs_RecordEvent = new(ModerationDefs_RecordEvent) + return json.Unmarshal(b, t.ModerationDefs_RecordEvent) + case "tools.ozone.moderation.defs#modEventPriorityScore": + t.ModerationDefs_ModEventPriorityScore = new(ModerationDefs_ModEventPriorityScore) + return json.Unmarshal(b, t.ModerationDefs_ModEventPriorityScore) + case "tools.ozone.moderation.defs#ageAssuranceEvent": + t.ModerationDefs_AgeAssuranceEvent = new(ModerationDefs_AgeAssuranceEvent) + return json.Unmarshal(b, t.ModerationDefs_AgeAssuranceEvent) + case "tools.ozone.moderation.defs#ageAssuranceOverrideEvent": + t.ModerationDefs_AgeAssuranceOverrideEvent = new(ModerationDefs_AgeAssuranceOverrideEvent) + return json.Unmarshal(b, t.ModerationDefs_AgeAssuranceOverrideEvent) + case "tools.ozone.moderation.defs#revokeAccountCredentialsEvent": + t.ModerationDefs_RevokeAccountCredentialsEvent = new(ModerationDefs_RevokeAccountCredentialsEvent) + return json.Unmarshal(b, t.ModerationDefs_RevokeAccountCredentialsEvent) + case "tools.ozone.moderation.defs#scheduleTakedownEvent": + t.ModerationDefs_ScheduleTakedownEvent = new(ModerationDefs_ScheduleTakedownEvent) + return json.Unmarshal(b, t.ModerationDefs_ScheduleTakedownEvent) + case "tools.ozone.moderation.defs#cancelScheduledTakedownEvent": + t.ModerationDefs_CancelScheduledTakedownEvent = new(ModerationDefs_CancelScheduledTakedownEvent) + return json.Unmarshal(b, t.ModerationDefs_CancelScheduledTakedownEvent) + default: + return nil + } +} + +type ModerationDefs_ModEventView_Subject struct { + AdminDefs_RepoRef *comatproto.AdminDefs_RepoRef + RepoStrongRef *comatproto.RepoStrongRef + ConvoDefs_MessageRef *chatbsky.ConvoDefs_MessageRef +} + +func (t *ModerationDefs_ModEventView_Subject) MarshalJSON() ([]byte, error) { + if t.AdminDefs_RepoRef != nil { + t.AdminDefs_RepoRef.LexiconTypeID = "com.atproto.admin.defs#repoRef" + return json.Marshal(t.AdminDefs_RepoRef) + } + if t.RepoStrongRef != nil { + t.RepoStrongRef.LexiconTypeID = "com.atproto.repo.strongRef" + return json.Marshal(t.RepoStrongRef) + } + if t.ConvoDefs_MessageRef != nil { + t.ConvoDefs_MessageRef.LexiconTypeID = "chat.bsky.convo.defs#messageRef" + return json.Marshal(t.ConvoDefs_MessageRef) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationDefs_ModEventView_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.admin.defs#repoRef": + t.AdminDefs_RepoRef = new(comatproto.AdminDefs_RepoRef) + return json.Unmarshal(b, t.AdminDefs_RepoRef) + case "com.atproto.repo.strongRef": + t.RepoStrongRef = new(comatproto.RepoStrongRef) + return json.Unmarshal(b, t.RepoStrongRef) + case "chat.bsky.convo.defs#messageRef": + t.ConvoDefs_MessageRef = new(chatbsky.ConvoDefs_MessageRef) + return json.Unmarshal(b, t.ConvoDefs_MessageRef) + default: + return nil + } +} + +// ModerationDefs_ModTool is a "modTool" in the tools.ozone.moderation.defs schema. +// +// Moderation tool information for tracing the source of the action +type ModerationDefs_ModTool struct { + // meta: Additional arbitrary metadata about the source + Meta *interface{} `json:"meta,omitempty" cborgen:"meta,omitempty"` + // name: Name/identifier of the source (e.g., 'automod', 'ozone/workspace') + Name string `json:"name" cborgen:"name"` +} + +// ModerationDefs_Moderation is a "moderation" in the tools.ozone.moderation.defs schema. +type ModerationDefs_Moderation struct { + SubjectStatus *ModerationDefs_SubjectStatusView `json:"subjectStatus,omitempty" cborgen:"subjectStatus,omitempty"` +} + +// ModerationDefs_ModerationDetail is a "moderationDetail" in the tools.ozone.moderation.defs schema. +type ModerationDefs_ModerationDetail struct { + SubjectStatus *ModerationDefs_SubjectStatusView `json:"subjectStatus,omitempty" cborgen:"subjectStatus,omitempty"` +} + +// ModerationDefs_RecordEvent is a "recordEvent" in the tools.ozone.moderation.defs schema. +// +// Logs lifecycle event on a record subject. Normally captured by automod from the firehose and emitted to ozone for historical tracking. +type ModerationDefs_RecordEvent struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#recordEvent"` + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + Op string `json:"op" cborgen:"op"` + Timestamp string `json:"timestamp" cborgen:"timestamp"` +} + +// ModerationDefs_RecordHosting is a "recordHosting" in the tools.ozone.moderation.defs schema. +type ModerationDefs_RecordHosting struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#recordHosting"` + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + DeletedAt *string `json:"deletedAt,omitempty" cborgen:"deletedAt,omitempty"` + Status string `json:"status" cborgen:"status"` + UpdatedAt *string `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"` +} + +// ModerationDefs_RecordView is a "recordView" in the tools.ozone.moderation.defs schema. +type ModerationDefs_RecordView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#recordView"` + BlobCids []string `json:"blobCids" cborgen:"blobCids"` + Cid string `json:"cid" cborgen:"cid"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Moderation *ModerationDefs_Moderation `json:"moderation" cborgen:"moderation"` + Repo *ModerationDefs_RepoView `json:"repo" cborgen:"repo"` + Uri string `json:"uri" cborgen:"uri"` + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` +} + +// ModerationDefs_RecordViewDetail is a "recordViewDetail" in the tools.ozone.moderation.defs schema. +type ModerationDefs_RecordViewDetail struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#recordViewDetail"` + Blobs []*ModerationDefs_BlobView `json:"blobs" cborgen:"blobs"` + Cid string `json:"cid" cborgen:"cid"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + Moderation *ModerationDefs_ModerationDetail `json:"moderation" cborgen:"moderation"` + Repo *ModerationDefs_RepoView `json:"repo" cborgen:"repo"` + Uri string `json:"uri" cborgen:"uri"` + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` +} + +// ModerationDefs_RecordViewNotFound is a "recordViewNotFound" in the tools.ozone.moderation.defs schema. +type ModerationDefs_RecordViewNotFound struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#recordViewNotFound"` + Uri string `json:"uri" cborgen:"uri"` +} + +// ModerationDefs_RecordsStats is a "recordsStats" in the tools.ozone.moderation.defs schema. +// +// Statistics about a set of record subject items +type ModerationDefs_RecordsStats struct { + // appealedCount: Number of items that were appealed at least once + AppealedCount *int64 `json:"appealedCount,omitempty" cborgen:"appealedCount,omitempty"` + // escalatedCount: Number of items that were escalated at least once + EscalatedCount *int64 `json:"escalatedCount,omitempty" cborgen:"escalatedCount,omitempty"` + // pendingCount: Number of item currently in "reviewOpen" or "reviewEscalated" state + PendingCount *int64 `json:"pendingCount,omitempty" cborgen:"pendingCount,omitempty"` + // processedCount: Number of item currently in "reviewNone" or "reviewClosed" state + ProcessedCount *int64 `json:"processedCount,omitempty" cborgen:"processedCount,omitempty"` + // reportedCount: Number of items that were reported at least once + ReportedCount *int64 `json:"reportedCount,omitempty" cborgen:"reportedCount,omitempty"` + // subjectCount: Total number of item in the set + SubjectCount *int64 `json:"subjectCount,omitempty" cborgen:"subjectCount,omitempty"` + // takendownCount: Number of item currently taken down + TakendownCount *int64 `json:"takendownCount,omitempty" cborgen:"takendownCount,omitempty"` + // totalReports: Cumulative sum of the number of reports on the items in the set + TotalReports *int64 `json:"totalReports,omitempty" cborgen:"totalReports,omitempty"` +} + +// ModerationDefs_RepoView is a "repoView" in the tools.ozone.moderation.defs schema. +type ModerationDefs_RepoView struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#repoView"` + DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` + Did string `json:"did" cborgen:"did"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` + InvitedBy *comatproto.ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` + InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` + Moderation *ModerationDefs_Moderation `json:"moderation" cborgen:"moderation"` + RelatedRecords []*lexutil.LexiconTypeDecoder `json:"relatedRecords" cborgen:"relatedRecords"` + ThreatSignatures []*comatproto.AdminDefs_ThreatSignature `json:"threatSignatures,omitempty" cborgen:"threatSignatures,omitempty"` +} + +// ModerationDefs_RepoViewDetail is a "repoViewDetail" in the tools.ozone.moderation.defs schema. +type ModerationDefs_RepoViewDetail struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#repoViewDetail"` + DeactivatedAt *string `json:"deactivatedAt,omitempty" cborgen:"deactivatedAt,omitempty"` + Did string `json:"did" cborgen:"did"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailConfirmedAt *string `json:"emailConfirmedAt,omitempty" cborgen:"emailConfirmedAt,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + IndexedAt string `json:"indexedAt" cborgen:"indexedAt"` + InviteNote *string `json:"inviteNote,omitempty" cborgen:"inviteNote,omitempty"` + InvitedBy *comatproto.ServerDefs_InviteCode `json:"invitedBy,omitempty" cborgen:"invitedBy,omitempty"` + Invites []*comatproto.ServerDefs_InviteCode `json:"invites,omitempty" cborgen:"invites,omitempty"` + InvitesDisabled *bool `json:"invitesDisabled,omitempty" cborgen:"invitesDisabled,omitempty"` + Labels []*comatproto.LabelDefs_Label `json:"labels,omitempty" cborgen:"labels,omitempty"` + Moderation *ModerationDefs_ModerationDetail `json:"moderation" cborgen:"moderation"` + RelatedRecords []*lexutil.LexiconTypeDecoder `json:"relatedRecords" cborgen:"relatedRecords"` + ThreatSignatures []*comatproto.AdminDefs_ThreatSignature `json:"threatSignatures,omitempty" cborgen:"threatSignatures,omitempty"` +} + +// ModerationDefs_RepoViewNotFound is a "repoViewNotFound" in the tools.ozone.moderation.defs schema. +type ModerationDefs_RepoViewNotFound struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#repoViewNotFound"` + Did string `json:"did" cborgen:"did"` +} + +// ModerationDefs_ReporterStats is a "reporterStats" in the tools.ozone.moderation.defs schema. +type ModerationDefs_ReporterStats struct { + // accountReportCount: The total number of reports made by the user on accounts. + AccountReportCount int64 `json:"accountReportCount" cborgen:"accountReportCount"` + Did string `json:"did" cborgen:"did"` + // labeledAccountCount: The total number of accounts labeled as a result of the user's reports. + LabeledAccountCount int64 `json:"labeledAccountCount" cborgen:"labeledAccountCount"` + // labeledRecordCount: The total number of records labeled as a result of the user's reports. + LabeledRecordCount int64 `json:"labeledRecordCount" cborgen:"labeledRecordCount"` + // recordReportCount: The total number of reports made by the user on records. + RecordReportCount int64 `json:"recordReportCount" cborgen:"recordReportCount"` + // reportedAccountCount: The total number of accounts reported by the user. + ReportedAccountCount int64 `json:"reportedAccountCount" cborgen:"reportedAccountCount"` + // reportedRecordCount: The total number of records reported by the user. + ReportedRecordCount int64 `json:"reportedRecordCount" cborgen:"reportedRecordCount"` + // takendownAccountCount: The total number of accounts taken down as a result of the user's reports. + TakendownAccountCount int64 `json:"takendownAccountCount" cborgen:"takendownAccountCount"` + // takendownRecordCount: The total number of records taken down as a result of the user's reports. + TakendownRecordCount int64 `json:"takendownRecordCount" cborgen:"takendownRecordCount"` +} + +// ModerationDefs_RevokeAccountCredentialsEvent is a "revokeAccountCredentialsEvent" in the tools.ozone.moderation.defs schema. +// +// Account credentials revocation by moderators. Only works on DID subjects. +type ModerationDefs_RevokeAccountCredentialsEvent struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#revokeAccountCredentialsEvent"` + // comment: Comment describing the reason for the revocation. + Comment string `json:"comment" cborgen:"comment"` +} + +// ModerationDefs_ScheduleTakedownEvent is a "scheduleTakedownEvent" in the tools.ozone.moderation.defs schema. +// +// Logs a scheduled takedown action for an account. +type ModerationDefs_ScheduleTakedownEvent struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#scheduleTakedownEvent"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + ExecuteAfter *string `json:"executeAfter,omitempty" cborgen:"executeAfter,omitempty"` + ExecuteAt *string `json:"executeAt,omitempty" cborgen:"executeAt,omitempty"` + ExecuteUntil *string `json:"executeUntil,omitempty" cborgen:"executeUntil,omitempty"` +} + +// ModerationDefs_ScheduledActionView is a "scheduledActionView" in the tools.ozone.moderation.defs schema. +// +// View of a scheduled moderation action +type ModerationDefs_ScheduledActionView struct { + // action: Type of action to be executed + Action string `json:"action" cborgen:"action"` + // createdAt: When the scheduled action was created + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // createdBy: DID of the user who created this scheduled action + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + // did: Subject DID for the action + Did string `json:"did" cborgen:"did"` + // eventData: Serialized event object that will be propagated to the event when performed + EventData *lexutil.LexiconTypeDecoder `json:"eventData,omitempty" cborgen:"eventData,omitempty"` + // executeAfter: Earliest time to execute the action (for randomized scheduling) + ExecuteAfter *string `json:"executeAfter,omitempty" cborgen:"executeAfter,omitempty"` + // executeAt: Exact time to execute the action + ExecuteAt *string `json:"executeAt,omitempty" cborgen:"executeAt,omitempty"` + // executeUntil: Latest time to execute the action (for randomized scheduling) + ExecuteUntil *string `json:"executeUntil,omitempty" cborgen:"executeUntil,omitempty"` + // executionEventId: ID of the moderation event created when action was successfully executed + ExecutionEventId *int64 `json:"executionEventId,omitempty" cborgen:"executionEventId,omitempty"` + // id: Auto-incrementing row ID + Id int64 `json:"id" cborgen:"id"` + // lastExecutedAt: When the action was last attempted to be executed + LastExecutedAt *string `json:"lastExecutedAt,omitempty" cborgen:"lastExecutedAt,omitempty"` + // lastFailureReason: Reason for the last execution failure + LastFailureReason *string `json:"lastFailureReason,omitempty" cborgen:"lastFailureReason,omitempty"` + // randomizeExecution: Whether execution time should be randomized within the specified range + RandomizeExecution *bool `json:"randomizeExecution,omitempty" cborgen:"randomizeExecution,omitempty"` + // status: Current status of the scheduled action + Status string `json:"status" cborgen:"status"` + // updatedAt: When the scheduled action was last updated + UpdatedAt *string `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"` +} + +// ModerationDefs_SubjectStatusView is a "subjectStatusView" in the tools.ozone.moderation.defs schema. +type ModerationDefs_SubjectStatusView struct { + // accountStats: Statistics related to the account subject + AccountStats *ModerationDefs_AccountStats `json:"accountStats,omitempty" cborgen:"accountStats,omitempty"` + // accountStrike: Strike information for the account (account-level only) + AccountStrike *ModerationDefs_AccountStrike `json:"accountStrike,omitempty" cborgen:"accountStrike,omitempty"` + // ageAssuranceState: Current age assurance state of the subject. + AgeAssuranceState *string `json:"ageAssuranceState,omitempty" cborgen:"ageAssuranceState,omitempty"` + // ageAssuranceUpdatedBy: Whether or not the last successful update to age assurance was made by the user or admin. + AgeAssuranceUpdatedBy *string `json:"ageAssuranceUpdatedBy,omitempty" cborgen:"ageAssuranceUpdatedBy,omitempty"` + // appealed: True indicates that the a previously taken moderator action was appealed against, by the author of the content. False indicates last appeal was resolved by moderators. + Appealed *bool `json:"appealed,omitempty" cborgen:"appealed,omitempty"` + // comment: Sticky comment on the subject. + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // createdAt: Timestamp referencing the first moderation status impacting event was emitted on the subject + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Hosting *ModerationDefs_SubjectStatusView_Hosting `json:"hosting,omitempty" cborgen:"hosting,omitempty"` + Id int64 `json:"id" cborgen:"id"` + // lastAppealedAt: Timestamp referencing when the author of the subject appealed a moderation action + LastAppealedAt *string `json:"lastAppealedAt,omitempty" cborgen:"lastAppealedAt,omitempty"` + LastReportedAt *string `json:"lastReportedAt,omitempty" cborgen:"lastReportedAt,omitempty"` + LastReviewedAt *string `json:"lastReviewedAt,omitempty" cborgen:"lastReviewedAt,omitempty"` + LastReviewedBy *string `json:"lastReviewedBy,omitempty" cborgen:"lastReviewedBy,omitempty"` + MuteReportingUntil *string `json:"muteReportingUntil,omitempty" cborgen:"muteReportingUntil,omitempty"` + MuteUntil *string `json:"muteUntil,omitempty" cborgen:"muteUntil,omitempty"` + // priorityScore: Numeric value representing the level of priority. Higher score means higher priority. + PriorityScore *int64 `json:"priorityScore,omitempty" cborgen:"priorityScore,omitempty"` + // recordsStats: Statistics related to the record subjects authored by the subject's account + RecordsStats *ModerationDefs_RecordsStats `json:"recordsStats,omitempty" cborgen:"recordsStats,omitempty"` + ReviewState *string `json:"reviewState" cborgen:"reviewState"` + Subject *ModerationDefs_SubjectStatusView_Subject `json:"subject" cborgen:"subject"` + SubjectBlobCids []string `json:"subjectBlobCids,omitempty" cborgen:"subjectBlobCids,omitempty"` + SubjectRepoHandle *string `json:"subjectRepoHandle,omitempty" cborgen:"subjectRepoHandle,omitempty"` + SuspendUntil *string `json:"suspendUntil,omitempty" cborgen:"suspendUntil,omitempty"` + Tags []string `json:"tags,omitempty" cborgen:"tags,omitempty"` + Takendown *bool `json:"takendown,omitempty" cborgen:"takendown,omitempty"` + // updatedAt: Timestamp referencing when the last update was made to the moderation status of the subject + UpdatedAt string `json:"updatedAt" cborgen:"updatedAt"` +} + +type ModerationDefs_SubjectStatusView_Hosting struct { + ModerationDefs_AccountHosting *ModerationDefs_AccountHosting + ModerationDefs_RecordHosting *ModerationDefs_RecordHosting +} + +func (t *ModerationDefs_SubjectStatusView_Hosting) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_AccountHosting != nil { + t.ModerationDefs_AccountHosting.LexiconTypeID = "tools.ozone.moderation.defs#accountHosting" + return json.Marshal(t.ModerationDefs_AccountHosting) + } + if t.ModerationDefs_RecordHosting != nil { + t.ModerationDefs_RecordHosting.LexiconTypeID = "tools.ozone.moderation.defs#recordHosting" + return json.Marshal(t.ModerationDefs_RecordHosting) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationDefs_SubjectStatusView_Hosting) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#accountHosting": + t.ModerationDefs_AccountHosting = new(ModerationDefs_AccountHosting) + return json.Unmarshal(b, t.ModerationDefs_AccountHosting) + case "tools.ozone.moderation.defs#recordHosting": + t.ModerationDefs_RecordHosting = new(ModerationDefs_RecordHosting) + return json.Unmarshal(b, t.ModerationDefs_RecordHosting) + default: + return nil + } +} + +type ModerationDefs_SubjectStatusView_Subject struct { + AdminDefs_RepoRef *comatproto.AdminDefs_RepoRef + RepoStrongRef *comatproto.RepoStrongRef + ConvoDefs_MessageRef *chatbsky.ConvoDefs_MessageRef +} + +func (t *ModerationDefs_SubjectStatusView_Subject) MarshalJSON() ([]byte, error) { + if t.AdminDefs_RepoRef != nil { + t.AdminDefs_RepoRef.LexiconTypeID = "com.atproto.admin.defs#repoRef" + return json.Marshal(t.AdminDefs_RepoRef) + } + if t.RepoStrongRef != nil { + t.RepoStrongRef.LexiconTypeID = "com.atproto.repo.strongRef" + return json.Marshal(t.RepoStrongRef) + } + if t.ConvoDefs_MessageRef != nil { + t.ConvoDefs_MessageRef.LexiconTypeID = "chat.bsky.convo.defs#messageRef" + return json.Marshal(t.ConvoDefs_MessageRef) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationDefs_SubjectStatusView_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.admin.defs#repoRef": + t.AdminDefs_RepoRef = new(comatproto.AdminDefs_RepoRef) + return json.Unmarshal(b, t.AdminDefs_RepoRef) + case "com.atproto.repo.strongRef": + t.RepoStrongRef = new(comatproto.RepoStrongRef) + return json.Unmarshal(b, t.RepoStrongRef) + case "chat.bsky.convo.defs#messageRef": + t.ConvoDefs_MessageRef = new(chatbsky.ConvoDefs_MessageRef) + return json.Unmarshal(b, t.ConvoDefs_MessageRef) + default: + return nil + } +} + +// ModerationDefs_SubjectView is a "subjectView" in the tools.ozone.moderation.defs schema. +// +// Detailed view of a subject. For record subjects, the author's repo and profile will be returned. +type ModerationDefs_SubjectView struct { + Profile *lexutil.LexiconTypeDecoder `json:"profile,omitempty" cborgen:"profile,omitempty"` + Record *ModerationDefs_RecordViewDetail `json:"record,omitempty" cborgen:"record,omitempty"` + Repo *ModerationDefs_RepoViewDetail `json:"repo,omitempty" cborgen:"repo,omitempty"` + Status *ModerationDefs_SubjectStatusView `json:"status,omitempty" cborgen:"status,omitempty"` + Subject string `json:"subject" cborgen:"subject"` + Type *string `json:"type" cborgen:"type"` +} + +// ModerationDefs_VideoDetails is a "videoDetails" in the tools.ozone.moderation.defs schema. +type ModerationDefs_VideoDetails struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.defs#videoDetails"` + Height int64 `json:"height" cborgen:"height"` + Length int64 `json:"length" cborgen:"length"` + Width int64 `json:"width" cborgen:"width"` +} diff --git a/api/ozone/moderationemitEvent.go b/api/ozone/moderationemitEvent.go new file mode 100644 index 000000000..38556ed59 --- /dev/null +++ b/api/ozone/moderationemitEvent.go @@ -0,0 +1,281 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.emitEvent + +package ozone + +import ( + "context" + "encoding/json" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationEmitEvent_Input is the input argument to a tools.ozone.moderation.emitEvent call. +type ModerationEmitEvent_Input struct { + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + Event *ModerationEmitEvent_Input_Event `json:"event" cborgen:"event"` + // externalId: An optional external ID for the event, used to deduplicate events from external systems. Fails when an event of same type with the same external ID exists for the same subject. + ExternalId *string `json:"externalId,omitempty" cborgen:"externalId,omitempty"` + ModTool *ModerationDefs_ModTool `json:"modTool,omitempty" cborgen:"modTool,omitempty"` + Subject *ModerationEmitEvent_Input_Subject `json:"subject" cborgen:"subject"` + SubjectBlobCids []string `json:"subjectBlobCids,omitempty" cborgen:"subjectBlobCids,omitempty"` +} + +type ModerationEmitEvent_Input_Event struct { + ModerationDefs_ModEventTakedown *ModerationDefs_ModEventTakedown + ModerationDefs_ModEventAcknowledge *ModerationDefs_ModEventAcknowledge + ModerationDefs_ModEventEscalate *ModerationDefs_ModEventEscalate + ModerationDefs_ModEventComment *ModerationDefs_ModEventComment + ModerationDefs_ModEventLabel *ModerationDefs_ModEventLabel + ModerationDefs_ModEventReport *ModerationDefs_ModEventReport + ModerationDefs_ModEventMute *ModerationDefs_ModEventMute + ModerationDefs_ModEventUnmute *ModerationDefs_ModEventUnmute + ModerationDefs_ModEventMuteReporter *ModerationDefs_ModEventMuteReporter + ModerationDefs_ModEventUnmuteReporter *ModerationDefs_ModEventUnmuteReporter + ModerationDefs_ModEventReverseTakedown *ModerationDefs_ModEventReverseTakedown + ModerationDefs_ModEventResolveAppeal *ModerationDefs_ModEventResolveAppeal + ModerationDefs_ModEventEmail *ModerationDefs_ModEventEmail + ModerationDefs_ModEventDivert *ModerationDefs_ModEventDivert + ModerationDefs_ModEventTag *ModerationDefs_ModEventTag + ModerationDefs_AccountEvent *ModerationDefs_AccountEvent + ModerationDefs_IdentityEvent *ModerationDefs_IdentityEvent + ModerationDefs_RecordEvent *ModerationDefs_RecordEvent + ModerationDefs_ModEventPriorityScore *ModerationDefs_ModEventPriorityScore + ModerationDefs_AgeAssuranceEvent *ModerationDefs_AgeAssuranceEvent + ModerationDefs_AgeAssuranceOverrideEvent *ModerationDefs_AgeAssuranceOverrideEvent + ModerationDefs_RevokeAccountCredentialsEvent *ModerationDefs_RevokeAccountCredentialsEvent + ModerationDefs_ScheduleTakedownEvent *ModerationDefs_ScheduleTakedownEvent + ModerationDefs_CancelScheduledTakedownEvent *ModerationDefs_CancelScheduledTakedownEvent +} + +func (t *ModerationEmitEvent_Input_Event) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_ModEventTakedown != nil { + t.ModerationDefs_ModEventTakedown.LexiconTypeID = "tools.ozone.moderation.defs#modEventTakedown" + return json.Marshal(t.ModerationDefs_ModEventTakedown) + } + if t.ModerationDefs_ModEventAcknowledge != nil { + t.ModerationDefs_ModEventAcknowledge.LexiconTypeID = "tools.ozone.moderation.defs#modEventAcknowledge" + return json.Marshal(t.ModerationDefs_ModEventAcknowledge) + } + if t.ModerationDefs_ModEventEscalate != nil { + t.ModerationDefs_ModEventEscalate.LexiconTypeID = "tools.ozone.moderation.defs#modEventEscalate" + return json.Marshal(t.ModerationDefs_ModEventEscalate) + } + if t.ModerationDefs_ModEventComment != nil { + t.ModerationDefs_ModEventComment.LexiconTypeID = "tools.ozone.moderation.defs#modEventComment" + return json.Marshal(t.ModerationDefs_ModEventComment) + } + if t.ModerationDefs_ModEventLabel != nil { + t.ModerationDefs_ModEventLabel.LexiconTypeID = "tools.ozone.moderation.defs#modEventLabel" + return json.Marshal(t.ModerationDefs_ModEventLabel) + } + if t.ModerationDefs_ModEventReport != nil { + t.ModerationDefs_ModEventReport.LexiconTypeID = "tools.ozone.moderation.defs#modEventReport" + return json.Marshal(t.ModerationDefs_ModEventReport) + } + if t.ModerationDefs_ModEventMute != nil { + t.ModerationDefs_ModEventMute.LexiconTypeID = "tools.ozone.moderation.defs#modEventMute" + return json.Marshal(t.ModerationDefs_ModEventMute) + } + if t.ModerationDefs_ModEventUnmute != nil { + t.ModerationDefs_ModEventUnmute.LexiconTypeID = "tools.ozone.moderation.defs#modEventUnmute" + return json.Marshal(t.ModerationDefs_ModEventUnmute) + } + if t.ModerationDefs_ModEventMuteReporter != nil { + t.ModerationDefs_ModEventMuteReporter.LexiconTypeID = "tools.ozone.moderation.defs#modEventMuteReporter" + return json.Marshal(t.ModerationDefs_ModEventMuteReporter) + } + if t.ModerationDefs_ModEventUnmuteReporter != nil { + t.ModerationDefs_ModEventUnmuteReporter.LexiconTypeID = "tools.ozone.moderation.defs#modEventUnmuteReporter" + return json.Marshal(t.ModerationDefs_ModEventUnmuteReporter) + } + if t.ModerationDefs_ModEventReverseTakedown != nil { + t.ModerationDefs_ModEventReverseTakedown.LexiconTypeID = "tools.ozone.moderation.defs#modEventReverseTakedown" + return json.Marshal(t.ModerationDefs_ModEventReverseTakedown) + } + if t.ModerationDefs_ModEventResolveAppeal != nil { + t.ModerationDefs_ModEventResolveAppeal.LexiconTypeID = "tools.ozone.moderation.defs#modEventResolveAppeal" + return json.Marshal(t.ModerationDefs_ModEventResolveAppeal) + } + if t.ModerationDefs_ModEventEmail != nil { + t.ModerationDefs_ModEventEmail.LexiconTypeID = "tools.ozone.moderation.defs#modEventEmail" + return json.Marshal(t.ModerationDefs_ModEventEmail) + } + if t.ModerationDefs_ModEventDivert != nil { + t.ModerationDefs_ModEventDivert.LexiconTypeID = "tools.ozone.moderation.defs#modEventDivert" + return json.Marshal(t.ModerationDefs_ModEventDivert) + } + if t.ModerationDefs_ModEventTag != nil { + t.ModerationDefs_ModEventTag.LexiconTypeID = "tools.ozone.moderation.defs#modEventTag" + return json.Marshal(t.ModerationDefs_ModEventTag) + } + if t.ModerationDefs_AccountEvent != nil { + t.ModerationDefs_AccountEvent.LexiconTypeID = "tools.ozone.moderation.defs#accountEvent" + return json.Marshal(t.ModerationDefs_AccountEvent) + } + if t.ModerationDefs_IdentityEvent != nil { + t.ModerationDefs_IdentityEvent.LexiconTypeID = "tools.ozone.moderation.defs#identityEvent" + return json.Marshal(t.ModerationDefs_IdentityEvent) + } + if t.ModerationDefs_RecordEvent != nil { + t.ModerationDefs_RecordEvent.LexiconTypeID = "tools.ozone.moderation.defs#recordEvent" + return json.Marshal(t.ModerationDefs_RecordEvent) + } + if t.ModerationDefs_ModEventPriorityScore != nil { + t.ModerationDefs_ModEventPriorityScore.LexiconTypeID = "tools.ozone.moderation.defs#modEventPriorityScore" + return json.Marshal(t.ModerationDefs_ModEventPriorityScore) + } + if t.ModerationDefs_AgeAssuranceEvent != nil { + t.ModerationDefs_AgeAssuranceEvent.LexiconTypeID = "tools.ozone.moderation.defs#ageAssuranceEvent" + return json.Marshal(t.ModerationDefs_AgeAssuranceEvent) + } + if t.ModerationDefs_AgeAssuranceOverrideEvent != nil { + t.ModerationDefs_AgeAssuranceOverrideEvent.LexiconTypeID = "tools.ozone.moderation.defs#ageAssuranceOverrideEvent" + return json.Marshal(t.ModerationDefs_AgeAssuranceOverrideEvent) + } + if t.ModerationDefs_RevokeAccountCredentialsEvent != nil { + t.ModerationDefs_RevokeAccountCredentialsEvent.LexiconTypeID = "tools.ozone.moderation.defs#revokeAccountCredentialsEvent" + return json.Marshal(t.ModerationDefs_RevokeAccountCredentialsEvent) + } + if t.ModerationDefs_ScheduleTakedownEvent != nil { + t.ModerationDefs_ScheduleTakedownEvent.LexiconTypeID = "tools.ozone.moderation.defs#scheduleTakedownEvent" + return json.Marshal(t.ModerationDefs_ScheduleTakedownEvent) + } + if t.ModerationDefs_CancelScheduledTakedownEvent != nil { + t.ModerationDefs_CancelScheduledTakedownEvent.LexiconTypeID = "tools.ozone.moderation.defs#cancelScheduledTakedownEvent" + return json.Marshal(t.ModerationDefs_CancelScheduledTakedownEvent) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationEmitEvent_Input_Event) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#modEventTakedown": + t.ModerationDefs_ModEventTakedown = new(ModerationDefs_ModEventTakedown) + return json.Unmarshal(b, t.ModerationDefs_ModEventTakedown) + case "tools.ozone.moderation.defs#modEventAcknowledge": + t.ModerationDefs_ModEventAcknowledge = new(ModerationDefs_ModEventAcknowledge) + return json.Unmarshal(b, t.ModerationDefs_ModEventAcknowledge) + case "tools.ozone.moderation.defs#modEventEscalate": + t.ModerationDefs_ModEventEscalate = new(ModerationDefs_ModEventEscalate) + return json.Unmarshal(b, t.ModerationDefs_ModEventEscalate) + case "tools.ozone.moderation.defs#modEventComment": + t.ModerationDefs_ModEventComment = new(ModerationDefs_ModEventComment) + return json.Unmarshal(b, t.ModerationDefs_ModEventComment) + case "tools.ozone.moderation.defs#modEventLabel": + t.ModerationDefs_ModEventLabel = new(ModerationDefs_ModEventLabel) + return json.Unmarshal(b, t.ModerationDefs_ModEventLabel) + case "tools.ozone.moderation.defs#modEventReport": + t.ModerationDefs_ModEventReport = new(ModerationDefs_ModEventReport) + return json.Unmarshal(b, t.ModerationDefs_ModEventReport) + case "tools.ozone.moderation.defs#modEventMute": + t.ModerationDefs_ModEventMute = new(ModerationDefs_ModEventMute) + return json.Unmarshal(b, t.ModerationDefs_ModEventMute) + case "tools.ozone.moderation.defs#modEventUnmute": + t.ModerationDefs_ModEventUnmute = new(ModerationDefs_ModEventUnmute) + return json.Unmarshal(b, t.ModerationDefs_ModEventUnmute) + case "tools.ozone.moderation.defs#modEventMuteReporter": + t.ModerationDefs_ModEventMuteReporter = new(ModerationDefs_ModEventMuteReporter) + return json.Unmarshal(b, t.ModerationDefs_ModEventMuteReporter) + case "tools.ozone.moderation.defs#modEventUnmuteReporter": + t.ModerationDefs_ModEventUnmuteReporter = new(ModerationDefs_ModEventUnmuteReporter) + return json.Unmarshal(b, t.ModerationDefs_ModEventUnmuteReporter) + case "tools.ozone.moderation.defs#modEventReverseTakedown": + t.ModerationDefs_ModEventReverseTakedown = new(ModerationDefs_ModEventReverseTakedown) + return json.Unmarshal(b, t.ModerationDefs_ModEventReverseTakedown) + case "tools.ozone.moderation.defs#modEventResolveAppeal": + t.ModerationDefs_ModEventResolveAppeal = new(ModerationDefs_ModEventResolveAppeal) + return json.Unmarshal(b, t.ModerationDefs_ModEventResolveAppeal) + case "tools.ozone.moderation.defs#modEventEmail": + t.ModerationDefs_ModEventEmail = new(ModerationDefs_ModEventEmail) + return json.Unmarshal(b, t.ModerationDefs_ModEventEmail) + case "tools.ozone.moderation.defs#modEventDivert": + t.ModerationDefs_ModEventDivert = new(ModerationDefs_ModEventDivert) + return json.Unmarshal(b, t.ModerationDefs_ModEventDivert) + case "tools.ozone.moderation.defs#modEventTag": + t.ModerationDefs_ModEventTag = new(ModerationDefs_ModEventTag) + return json.Unmarshal(b, t.ModerationDefs_ModEventTag) + case "tools.ozone.moderation.defs#accountEvent": + t.ModerationDefs_AccountEvent = new(ModerationDefs_AccountEvent) + return json.Unmarshal(b, t.ModerationDefs_AccountEvent) + case "tools.ozone.moderation.defs#identityEvent": + t.ModerationDefs_IdentityEvent = new(ModerationDefs_IdentityEvent) + return json.Unmarshal(b, t.ModerationDefs_IdentityEvent) + case "tools.ozone.moderation.defs#recordEvent": + t.ModerationDefs_RecordEvent = new(ModerationDefs_RecordEvent) + return json.Unmarshal(b, t.ModerationDefs_RecordEvent) + case "tools.ozone.moderation.defs#modEventPriorityScore": + t.ModerationDefs_ModEventPriorityScore = new(ModerationDefs_ModEventPriorityScore) + return json.Unmarshal(b, t.ModerationDefs_ModEventPriorityScore) + case "tools.ozone.moderation.defs#ageAssuranceEvent": + t.ModerationDefs_AgeAssuranceEvent = new(ModerationDefs_AgeAssuranceEvent) + return json.Unmarshal(b, t.ModerationDefs_AgeAssuranceEvent) + case "tools.ozone.moderation.defs#ageAssuranceOverrideEvent": + t.ModerationDefs_AgeAssuranceOverrideEvent = new(ModerationDefs_AgeAssuranceOverrideEvent) + return json.Unmarshal(b, t.ModerationDefs_AgeAssuranceOverrideEvent) + case "tools.ozone.moderation.defs#revokeAccountCredentialsEvent": + t.ModerationDefs_RevokeAccountCredentialsEvent = new(ModerationDefs_RevokeAccountCredentialsEvent) + return json.Unmarshal(b, t.ModerationDefs_RevokeAccountCredentialsEvent) + case "tools.ozone.moderation.defs#scheduleTakedownEvent": + t.ModerationDefs_ScheduleTakedownEvent = new(ModerationDefs_ScheduleTakedownEvent) + return json.Unmarshal(b, t.ModerationDefs_ScheduleTakedownEvent) + case "tools.ozone.moderation.defs#cancelScheduledTakedownEvent": + t.ModerationDefs_CancelScheduledTakedownEvent = new(ModerationDefs_CancelScheduledTakedownEvent) + return json.Unmarshal(b, t.ModerationDefs_CancelScheduledTakedownEvent) + default: + return nil + } +} + +type ModerationEmitEvent_Input_Subject struct { + AdminDefs_RepoRef *comatproto.AdminDefs_RepoRef + RepoStrongRef *comatproto.RepoStrongRef +} + +func (t *ModerationEmitEvent_Input_Subject) MarshalJSON() ([]byte, error) { + if t.AdminDefs_RepoRef != nil { + t.AdminDefs_RepoRef.LexiconTypeID = "com.atproto.admin.defs#repoRef" + return json.Marshal(t.AdminDefs_RepoRef) + } + if t.RepoStrongRef != nil { + t.RepoStrongRef.LexiconTypeID = "com.atproto.repo.strongRef" + return json.Marshal(t.RepoStrongRef) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationEmitEvent_Input_Subject) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "com.atproto.admin.defs#repoRef": + t.AdminDefs_RepoRef = new(comatproto.AdminDefs_RepoRef) + return json.Unmarshal(b, t.AdminDefs_RepoRef) + case "com.atproto.repo.strongRef": + t.RepoStrongRef = new(comatproto.RepoStrongRef) + return json.Unmarshal(b, t.RepoStrongRef) + default: + return nil + } +} + +// ModerationEmitEvent calls the XRPC method "tools.ozone.moderation.emitEvent". +func ModerationEmitEvent(ctx context.Context, c lexutil.LexClient, input *ModerationEmitEvent_Input) (*ModerationDefs_ModEventView, error) { + var out ModerationDefs_ModEventView + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.moderation.emitEvent", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetAccountTimeline.go b/api/ozone/moderationgetAccountTimeline.go new file mode 100644 index 000000000..3e86c3c53 --- /dev/null +++ b/api/ozone/moderationgetAccountTimeline.go @@ -0,0 +1,42 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.getAccountTimeline + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetAccountTimeline_Output is the output of a tools.ozone.moderation.getAccountTimeline call. +type ModerationGetAccountTimeline_Output struct { + Timeline []*ModerationGetAccountTimeline_TimelineItem `json:"timeline" cborgen:"timeline"` +} + +// ModerationGetAccountTimeline_TimelineItem is a "timelineItem" in the tools.ozone.moderation.getAccountTimeline schema. +type ModerationGetAccountTimeline_TimelineItem struct { + Day string `json:"day" cborgen:"day"` + Summary []*ModerationGetAccountTimeline_TimelineItemSummary `json:"summary" cborgen:"summary"` +} + +// ModerationGetAccountTimeline_TimelineItemSummary is a "timelineItemSummary" in the tools.ozone.moderation.getAccountTimeline schema. +type ModerationGetAccountTimeline_TimelineItemSummary struct { + Count int64 `json:"count" cborgen:"count"` + EventSubjectType string `json:"eventSubjectType" cborgen:"eventSubjectType"` + EventType string `json:"eventType" cborgen:"eventType"` +} + +// ModerationGetAccountTimeline calls the XRPC method "tools.ozone.moderation.getAccountTimeline". +func ModerationGetAccountTimeline(ctx context.Context, c lexutil.LexClient, did string) (*ModerationGetAccountTimeline_Output, error) { + var out ModerationGetAccountTimeline_Output + + params := map[string]interface{}{} + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.getAccountTimeline", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetEvent.go b/api/ozone/moderationgetEvent.go new file mode 100644 index 000000000..c9ed3a6fd --- /dev/null +++ b/api/ozone/moderationgetEvent.go @@ -0,0 +1,24 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.getEvent + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetEvent calls the XRPC method "tools.ozone.moderation.getEvent". +func ModerationGetEvent(ctx context.Context, c lexutil.LexClient, id int64) (*ModerationDefs_ModEventViewDetail, error) { + var out ModerationDefs_ModEventViewDetail + + params := map[string]interface{}{} + params["id"] = id + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.getEvent", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetRecord.go b/api/ozone/moderationgetRecord.go new file mode 100644 index 000000000..3dc38735f --- /dev/null +++ b/api/ozone/moderationgetRecord.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.getRecord + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetRecord calls the XRPC method "tools.ozone.moderation.getRecord". +func ModerationGetRecord(ctx context.Context, c lexutil.LexClient, cid string, uri string) (*ModerationDefs_RecordViewDetail, error) { + var out ModerationDefs_RecordViewDetail + + params := map[string]interface{}{} + if cid != "" { + params["cid"] = cid + } + params["uri"] = uri + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.getRecord", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetRecords.go b/api/ozone/moderationgetRecords.go new file mode 100644 index 000000000..79db28854 --- /dev/null +++ b/api/ozone/moderationgetRecords.go @@ -0,0 +1,66 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.getRecords + +package ozone + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetRecords_Output is the output of a tools.ozone.moderation.getRecords call. +type ModerationGetRecords_Output struct { + Records []*ModerationGetRecords_Output_Records_Elem `json:"records" cborgen:"records"` +} + +type ModerationGetRecords_Output_Records_Elem struct { + ModerationDefs_RecordViewDetail *ModerationDefs_RecordViewDetail + ModerationDefs_RecordViewNotFound *ModerationDefs_RecordViewNotFound +} + +func (t *ModerationGetRecords_Output_Records_Elem) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_RecordViewDetail != nil { + t.ModerationDefs_RecordViewDetail.LexiconTypeID = "tools.ozone.moderation.defs#recordViewDetail" + return json.Marshal(t.ModerationDefs_RecordViewDetail) + } + if t.ModerationDefs_RecordViewNotFound != nil { + t.ModerationDefs_RecordViewNotFound.LexiconTypeID = "tools.ozone.moderation.defs#recordViewNotFound" + return json.Marshal(t.ModerationDefs_RecordViewNotFound) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationGetRecords_Output_Records_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#recordViewDetail": + t.ModerationDefs_RecordViewDetail = new(ModerationDefs_RecordViewDetail) + return json.Unmarshal(b, t.ModerationDefs_RecordViewDetail) + case "tools.ozone.moderation.defs#recordViewNotFound": + t.ModerationDefs_RecordViewNotFound = new(ModerationDefs_RecordViewNotFound) + return json.Unmarshal(b, t.ModerationDefs_RecordViewNotFound) + default: + return nil + } +} + +// ModerationGetRecords calls the XRPC method "tools.ozone.moderation.getRecords". +func ModerationGetRecords(ctx context.Context, c lexutil.LexClient, uris []string) (*ModerationGetRecords_Output, error) { + var out ModerationGetRecords_Output + + params := map[string]interface{}{} + params["uris"] = uris + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.getRecords", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetRepo.go b/api/ozone/moderationgetRepo.go new file mode 100644 index 000000000..77148ee86 --- /dev/null +++ b/api/ozone/moderationgetRepo.go @@ -0,0 +1,24 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.getRepo + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetRepo calls the XRPC method "tools.ozone.moderation.getRepo". +func ModerationGetRepo(ctx context.Context, c lexutil.LexClient, did string) (*ModerationDefs_RepoViewDetail, error) { + var out ModerationDefs_RepoViewDetail + + params := map[string]interface{}{} + params["did"] = did + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.getRepo", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetReporterStats.go b/api/ozone/moderationgetReporterStats.go new file mode 100644 index 000000000..796935096 --- /dev/null +++ b/api/ozone/moderationgetReporterStats.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.getReporterStats + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetReporterStats_Output is the output of a tools.ozone.moderation.getReporterStats call. +type ModerationGetReporterStats_Output struct { + Stats []*ModerationDefs_ReporterStats `json:"stats" cborgen:"stats"` +} + +// ModerationGetReporterStats calls the XRPC method "tools.ozone.moderation.getReporterStats". +func ModerationGetReporterStats(ctx context.Context, c lexutil.LexClient, dids []string) (*ModerationGetReporterStats_Output, error) { + var out ModerationGetReporterStats_Output + + params := map[string]interface{}{} + params["dids"] = dids + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.getReporterStats", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetRepos.go b/api/ozone/moderationgetRepos.go new file mode 100644 index 000000000..d96f7e2dc --- /dev/null +++ b/api/ozone/moderationgetRepos.go @@ -0,0 +1,66 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.getRepos + +package ozone + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetRepos_Output is the output of a tools.ozone.moderation.getRepos call. +type ModerationGetRepos_Output struct { + Repos []*ModerationGetRepos_Output_Repos_Elem `json:"repos" cborgen:"repos"` +} + +type ModerationGetRepos_Output_Repos_Elem struct { + ModerationDefs_RepoViewDetail *ModerationDefs_RepoViewDetail + ModerationDefs_RepoViewNotFound *ModerationDefs_RepoViewNotFound +} + +func (t *ModerationGetRepos_Output_Repos_Elem) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_RepoViewDetail != nil { + t.ModerationDefs_RepoViewDetail.LexiconTypeID = "tools.ozone.moderation.defs#repoViewDetail" + return json.Marshal(t.ModerationDefs_RepoViewDetail) + } + if t.ModerationDefs_RepoViewNotFound != nil { + t.ModerationDefs_RepoViewNotFound.LexiconTypeID = "tools.ozone.moderation.defs#repoViewNotFound" + return json.Marshal(t.ModerationDefs_RepoViewNotFound) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationGetRepos_Output_Repos_Elem) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#repoViewDetail": + t.ModerationDefs_RepoViewDetail = new(ModerationDefs_RepoViewDetail) + return json.Unmarshal(b, t.ModerationDefs_RepoViewDetail) + case "tools.ozone.moderation.defs#repoViewNotFound": + t.ModerationDefs_RepoViewNotFound = new(ModerationDefs_RepoViewNotFound) + return json.Unmarshal(b, t.ModerationDefs_RepoViewNotFound) + default: + return nil + } +} + +// ModerationGetRepos calls the XRPC method "tools.ozone.moderation.getRepos". +func ModerationGetRepos(ctx context.Context, c lexutil.LexClient, dids []string) (*ModerationGetRepos_Output, error) { + var out ModerationGetRepos_Output + + params := map[string]interface{}{} + params["dids"] = dids + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.getRepos", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationgetSubjects.go b/api/ozone/moderationgetSubjects.go new file mode 100644 index 000000000..c739d6cf7 --- /dev/null +++ b/api/ozone/moderationgetSubjects.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.getSubjects + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationGetSubjects_Output is the output of a tools.ozone.moderation.getSubjects call. +type ModerationGetSubjects_Output struct { + Subjects []*ModerationDefs_SubjectView `json:"subjects" cborgen:"subjects"` +} + +// ModerationGetSubjects calls the XRPC method "tools.ozone.moderation.getSubjects". +func ModerationGetSubjects(ctx context.Context, c lexutil.LexClient, subjects []string) (*ModerationGetSubjects_Output, error) { + var out ModerationGetSubjects_Output + + params := map[string]interface{}{} + params["subjects"] = subjects + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.getSubjects", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationlistScheduledActions.go b/api/ozone/moderationlistScheduledActions.go new file mode 100644 index 000000000..6fb6276ef --- /dev/null +++ b/api/ozone/moderationlistScheduledActions.go @@ -0,0 +1,44 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.listScheduledActions + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationListScheduledActions_Input is the input argument to a tools.ozone.moderation.listScheduledActions call. +type ModerationListScheduledActions_Input struct { + // cursor: Cursor for pagination + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // endsBefore: Filter actions scheduled to execute before this time + EndsBefore *string `json:"endsBefore,omitempty" cborgen:"endsBefore,omitempty"` + // limit: Maximum number of results to return + Limit *int64 `json:"limit,omitempty" cborgen:"limit,omitempty"` + // startsAfter: Filter actions scheduled to execute after this time + StartsAfter *string `json:"startsAfter,omitempty" cborgen:"startsAfter,omitempty"` + // statuses: Filter actions by status + Statuses []string `json:"statuses" cborgen:"statuses"` + // subjects: Filter actions for specific DID subjects + Subjects []string `json:"subjects,omitempty" cborgen:"subjects,omitempty"` +} + +// ModerationListScheduledActions_Output is the output of a tools.ozone.moderation.listScheduledActions call. +type ModerationListScheduledActions_Output struct { + Actions []*ModerationDefs_ScheduledActionView `json:"actions" cborgen:"actions"` + // cursor: Cursor for next page of results + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// ModerationListScheduledActions calls the XRPC method "tools.ozone.moderation.listScheduledActions". +func ModerationListScheduledActions(ctx context.Context, c lexutil.LexClient, input *ModerationListScheduledActions_Input) (*ModerationListScheduledActions_Output, error) { + var out ModerationListScheduledActions_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.moderation.listScheduledActions", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationqueryEvents.go b/api/ozone/moderationqueryEvents.go new file mode 100644 index 000000000..bfd6bc31d --- /dev/null +++ b/api/ozone/moderationqueryEvents.go @@ -0,0 +1,116 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.queryEvents + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationQueryEvents_Output is the output of a tools.ozone.moderation.queryEvents call. +type ModerationQueryEvents_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Events []*ModerationDefs_ModEventView `json:"events" cborgen:"events"` +} + +// ModerationQueryEvents calls the XRPC method "tools.ozone.moderation.queryEvents". +// +// addedLabels: If specified, only events where all of these labels were added are returned +// addedTags: If specified, only events where all of these tags were added are returned +// ageAssuranceState: If specified, only events where the age assurance state matches the given value are returned +// batchId: If specified, only events where the batchId matches the given value are returned +// collections: If specified, only events where the subject belongs to the given collections will be returned. When subjectType is set to 'account', this will be ignored. +// comment: If specified, only events with comments containing the keyword are returned. Apply || separator to use multiple keywords and match using OR condition. +// createdAfter: Retrieve events created after a given timestamp +// createdBefore: Retrieve events created before a given timestamp +// hasComment: If true, only events with comments are returned +// includeAllUserRecords: If true, events on all record types (posts, lists, profile etc.) or records from given 'collections' param, owned by the did are returned. +// modTool: If specified, only events where the modTool name matches any of the given values are returned +// removedLabels: If specified, only events where all of these labels were removed are returned +// removedTags: If specified, only events where all of these tags were removed are returned +// sortDirection: Sort direction for the events. Defaults to descending order of created at timestamp. +// subjectType: If specified, only events where the subject is of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. +// types: The types of events (fully qualified string in the format of tools.ozone.moderation.defs#modEvent) to filter by. If not specified, all events are returned. +// withStrike: If specified, only events where strikeCount value is set are returned. +func ModerationQueryEvents(ctx context.Context, c lexutil.LexClient, addedLabels []string, addedTags []string, ageAssuranceState string, batchId string, collections []string, comment string, createdAfter string, createdBefore string, createdBy string, cursor string, hasComment bool, includeAllUserRecords bool, limit int64, modTool []string, policies []string, removedLabels []string, removedTags []string, reportTypes []string, sortDirection string, subject string, subjectType string, types []string, withStrike bool) (*ModerationQueryEvents_Output, error) { + var out ModerationQueryEvents_Output + + params := map[string]interface{}{} + if len(addedLabels) != 0 { + params["addedLabels"] = addedLabels + } + if len(addedTags) != 0 { + params["addedTags"] = addedTags + } + if ageAssuranceState != "" { + params["ageAssuranceState"] = ageAssuranceState + } + if batchId != "" { + params["batchId"] = batchId + } + if len(collections) != 0 { + params["collections"] = collections + } + if comment != "" { + params["comment"] = comment + } + if createdAfter != "" { + params["createdAfter"] = createdAfter + } + if createdBefore != "" { + params["createdBefore"] = createdBefore + } + if createdBy != "" { + params["createdBy"] = createdBy + } + if cursor != "" { + params["cursor"] = cursor + } + if hasComment { + params["hasComment"] = hasComment + } + if includeAllUserRecords { + params["includeAllUserRecords"] = includeAllUserRecords + } + if limit != 0 { + params["limit"] = limit + } + if len(modTool) != 0 { + params["modTool"] = modTool + } + if len(policies) != 0 { + params["policies"] = policies + } + if len(removedLabels) != 0 { + params["removedLabels"] = removedLabels + } + if len(removedTags) != 0 { + params["removedTags"] = removedTags + } + if len(reportTypes) != 0 { + params["reportTypes"] = reportTypes + } + if sortDirection != "" { + params["sortDirection"] = sortDirection + } + if subject != "" { + params["subject"] = subject + } + if subjectType != "" { + params["subjectType"] = subjectType + } + if len(types) != 0 { + params["types"] = types + } + if withStrike { + params["withStrike"] = withStrike + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.queryEvents", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationqueryStatuses.go b/api/ozone/moderationqueryStatuses.go new file mode 100644 index 000000000..b455f3f3c --- /dev/null +++ b/api/ozone/moderationqueryStatuses.go @@ -0,0 +1,167 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.queryStatuses + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationQueryStatuses_Output is the output of a tools.ozone.moderation.queryStatuses call. +type ModerationQueryStatuses_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + SubjectStatuses []*ModerationDefs_SubjectStatusView `json:"subjectStatuses" cborgen:"subjectStatuses"` +} + +// ModerationQueryStatuses calls the XRPC method "tools.ozone.moderation.queryStatuses". +// +// ageAssuranceState: If specified, only subjects with the given age assurance state will be returned. +// appealed: Get subjects in unresolved appealed status +// collections: If specified, subjects belonging to the given collections will be returned. When subjectType is set to 'account', this will be ignored. +// comment: Search subjects by keyword from comments +// hostingDeletedAfter: Search subjects where the associated record/account was deleted after a given timestamp +// hostingDeletedBefore: Search subjects where the associated record/account was deleted before a given timestamp +// hostingStatuses: Search subjects by the status of the associated record/account +// hostingUpdatedAfter: Search subjects where the associated record/account was updated after a given timestamp +// hostingUpdatedBefore: Search subjects where the associated record/account was updated before a given timestamp +// includeAllUserRecords: All subjects, or subjects from given 'collections' param, belonging to the account specified in the 'subject' param will be returned. +// includeMuted: By default, we don't include muted subjects in the results. Set this to true to include them. +// lastReviewedBy: Get all subject statuses that were reviewed by a specific moderator +// minAccountSuspendCount: If specified, only subjects that belong to an account that has at least this many suspensions will be returned. +// minPriorityScore: If specified, only subjects that have priority score value above the given value will be returned. +// minReportedRecordsCount: If specified, only subjects that belong to an account that has at least this many reported records will be returned. +// minStrikeCount: If specified, only subjects that belong to an account that has at least this many active strikes will be returned. +// minTakendownRecordsCount: If specified, only subjects that belong to an account that has at least this many taken down records will be returned. +// onlyMuted: When set to true, only muted subjects and reporters will be returned. +// queueCount: Number of queues being used by moderators. Subjects will be split among all queues. +// queueIndex: Index of the queue to fetch subjects from. Works only when queueCount value is specified. +// queueSeed: A seeder to shuffle/balance the queue items. +// reportedAfter: Search subjects reported after a given timestamp +// reportedBefore: Search subjects reported before a given timestamp +// reviewState: Specify when fetching subjects in a certain state +// reviewedAfter: Search subjects reviewed after a given timestamp +// reviewedBefore: Search subjects reviewed before a given timestamp +// subject: The subject to get the status for. +// subjectType: If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. +// takendown: Get subjects that were taken down +func ModerationQueryStatuses(ctx context.Context, c lexutil.LexClient, ageAssuranceState string, appealed bool, collections []string, comment string, cursor string, excludeTags []string, hostingDeletedAfter string, hostingDeletedBefore string, hostingStatuses []string, hostingUpdatedAfter string, hostingUpdatedBefore string, ignoreSubjects []string, includeAllUserRecords bool, includeMuted bool, lastReviewedBy string, limit int64, minAccountSuspendCount int64, minPriorityScore int64, minReportedRecordsCount int64, minStrikeCount int64, minTakendownRecordsCount int64, onlyMuted bool, queueCount int64, queueIndex int64, queueSeed string, reportedAfter string, reportedBefore string, reviewState string, reviewedAfter string, reviewedBefore string, sortDirection string, sortField string, subject string, subjectType string, tags []string, takendown bool) (*ModerationQueryStatuses_Output, error) { + var out ModerationQueryStatuses_Output + + params := map[string]interface{}{} + if ageAssuranceState != "" { + params["ageAssuranceState"] = ageAssuranceState + } + if appealed { + params["appealed"] = appealed + } + if len(collections) != 0 { + params["collections"] = collections + } + if comment != "" { + params["comment"] = comment + } + if cursor != "" { + params["cursor"] = cursor + } + if len(excludeTags) != 0 { + params["excludeTags"] = excludeTags + } + if hostingDeletedAfter != "" { + params["hostingDeletedAfter"] = hostingDeletedAfter + } + if hostingDeletedBefore != "" { + params["hostingDeletedBefore"] = hostingDeletedBefore + } + if len(hostingStatuses) != 0 { + params["hostingStatuses"] = hostingStatuses + } + if hostingUpdatedAfter != "" { + params["hostingUpdatedAfter"] = hostingUpdatedAfter + } + if hostingUpdatedBefore != "" { + params["hostingUpdatedBefore"] = hostingUpdatedBefore + } + if len(ignoreSubjects) != 0 { + params["ignoreSubjects"] = ignoreSubjects + } + if includeAllUserRecords { + params["includeAllUserRecords"] = includeAllUserRecords + } + if includeMuted { + params["includeMuted"] = includeMuted + } + if lastReviewedBy != "" { + params["lastReviewedBy"] = lastReviewedBy + } + if limit != 0 { + params["limit"] = limit + } + if minAccountSuspendCount != 0 { + params["minAccountSuspendCount"] = minAccountSuspendCount + } + if minPriorityScore != 0 { + params["minPriorityScore"] = minPriorityScore + } + if minReportedRecordsCount != 0 { + params["minReportedRecordsCount"] = minReportedRecordsCount + } + if minStrikeCount != 0 { + params["minStrikeCount"] = minStrikeCount + } + if minTakendownRecordsCount != 0 { + params["minTakendownRecordsCount"] = minTakendownRecordsCount + } + if onlyMuted { + params["onlyMuted"] = onlyMuted + } + if queueCount != 0 { + params["queueCount"] = queueCount + } + if queueIndex != 0 { + params["queueIndex"] = queueIndex + } + if queueSeed != "" { + params["queueSeed"] = queueSeed + } + if reportedAfter != "" { + params["reportedAfter"] = reportedAfter + } + if reportedBefore != "" { + params["reportedBefore"] = reportedBefore + } + if reviewState != "" { + params["reviewState"] = reviewState + } + if reviewedAfter != "" { + params["reviewedAfter"] = reviewedAfter + } + if reviewedBefore != "" { + params["reviewedBefore"] = reviewedBefore + } + if sortDirection != "" { + params["sortDirection"] = sortDirection + } + if sortField != "" { + params["sortField"] = sortField + } + if subject != "" { + params["subject"] = subject + } + if subjectType != "" { + params["subjectType"] = subjectType + } + if len(tags) != 0 { + params["tags"] = tags + } + if takendown { + params["takendown"] = takendown + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.queryStatuses", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationscheduleAction.go b/api/ozone/moderationscheduleAction.go new file mode 100644 index 000000000..728fdf5ac --- /dev/null +++ b/api/ozone/moderationscheduleAction.go @@ -0,0 +1,110 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.scheduleAction + +package ozone + +import ( + "context" + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationScheduleAction_FailedScheduling is a "failedScheduling" in the tools.ozone.moderation.scheduleAction schema. +type ModerationScheduleAction_FailedScheduling struct { + Error string `json:"error" cborgen:"error"` + ErrorCode *string `json:"errorCode,omitempty" cborgen:"errorCode,omitempty"` + Subject string `json:"subject" cborgen:"subject"` +} + +// ModerationScheduleAction_Input is the input argument to a tools.ozone.moderation.scheduleAction call. +type ModerationScheduleAction_Input struct { + Action *ModerationScheduleAction_Input_Action `json:"action" cborgen:"action"` + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + // modTool: This will be propagated to the moderation event when it is applied + ModTool *ModerationDefs_ModTool `json:"modTool,omitempty" cborgen:"modTool,omitempty"` + Scheduling *ModerationScheduleAction_SchedulingConfig `json:"scheduling" cborgen:"scheduling"` + // subjects: Array of DID subjects to schedule the action for + Subjects []string `json:"subjects" cborgen:"subjects"` +} + +type ModerationScheduleAction_Input_Action struct { + ModerationScheduleAction_Takedown *ModerationScheduleAction_Takedown +} + +func (t *ModerationScheduleAction_Input_Action) MarshalJSON() ([]byte, error) { + if t.ModerationScheduleAction_Takedown != nil { + t.ModerationScheduleAction_Takedown.LexiconTypeID = "tools.ozone.moderation.scheduleAction#takedown" + return json.Marshal(t.ModerationScheduleAction_Takedown) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *ModerationScheduleAction_Input_Action) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.scheduleAction#takedown": + t.ModerationScheduleAction_Takedown = new(ModerationScheduleAction_Takedown) + return json.Unmarshal(b, t.ModerationScheduleAction_Takedown) + default: + return nil + } +} + +// ModerationScheduleAction_ScheduledActionResults is a "scheduledActionResults" in the tools.ozone.moderation.scheduleAction schema. +type ModerationScheduleAction_ScheduledActionResults struct { + Failed []*ModerationScheduleAction_FailedScheduling `json:"failed" cborgen:"failed"` + Succeeded []string `json:"succeeded" cborgen:"succeeded"` +} + +// ModerationScheduleAction_SchedulingConfig is a "schedulingConfig" in the tools.ozone.moderation.scheduleAction schema. +// +// Configuration for when the action should be executed +type ModerationScheduleAction_SchedulingConfig struct { + // executeAfter: Earliest time to execute the action (for randomized scheduling) + ExecuteAfter *string `json:"executeAfter,omitempty" cborgen:"executeAfter,omitempty"` + // executeAt: Exact time to execute the action + ExecuteAt *string `json:"executeAt,omitempty" cborgen:"executeAt,omitempty"` + // executeUntil: Latest time to execute the action (for randomized scheduling) + ExecuteUntil *string `json:"executeUntil,omitempty" cborgen:"executeUntil,omitempty"` +} + +// ModerationScheduleAction_Takedown is a "takedown" in the tools.ozone.moderation.scheduleAction schema. +// +// Schedule a takedown action +type ModerationScheduleAction_Takedown struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=tools.ozone.moderation.scheduleAction#takedown"` + // acknowledgeAccountSubjects: If true, all other reports on content authored by this account will be resolved (acknowledged). + AcknowledgeAccountSubjects *bool `json:"acknowledgeAccountSubjects,omitempty" cborgen:"acknowledgeAccountSubjects,omitempty"` + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // durationInHours: Indicates how long the takedown should be in effect before automatically expiring. + DurationInHours *int64 `json:"durationInHours,omitempty" cborgen:"durationInHours,omitempty"` + // emailContent: Email content to be sent to the user upon takedown. + EmailContent *string `json:"emailContent,omitempty" cborgen:"emailContent,omitempty"` + // emailSubject: Subject of the email to be sent to the user upon takedown. + EmailSubject *string `json:"emailSubject,omitempty" cborgen:"emailSubject,omitempty"` + // policies: Names/Keywords of the policies that drove the decision. + Policies []string `json:"policies,omitempty" cborgen:"policies,omitempty"` + // severityLevel: Severity level of the violation (e.g., 'sev-0', 'sev-1', 'sev-2', etc.). + SeverityLevel *string `json:"severityLevel,omitempty" cborgen:"severityLevel,omitempty"` + // strikeCount: Number of strikes to assign to the user when takedown is applied. + StrikeCount *int64 `json:"strikeCount,omitempty" cborgen:"strikeCount,omitempty"` + // strikeExpiresAt: When the strike should expire. If not provided, the strike never expires. + StrikeExpiresAt *string `json:"strikeExpiresAt,omitempty" cborgen:"strikeExpiresAt,omitempty"` +} + +// ModerationScheduleAction calls the XRPC method "tools.ozone.moderation.scheduleAction". +func ModerationScheduleAction(ctx context.Context, c lexutil.LexClient, input *ModerationScheduleAction_Input) (*ModerationScheduleAction_ScheduledActionResults, error) { + var out ModerationScheduleAction_ScheduledActionResults + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.moderation.scheduleAction", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/moderationsearchRepos.go b/api/ozone/moderationsearchRepos.go new file mode 100644 index 000000000..184926b4f --- /dev/null +++ b/api/ozone/moderationsearchRepos.go @@ -0,0 +1,43 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.moderation.searchRepos + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ModerationSearchRepos_Output is the output of a tools.ozone.moderation.searchRepos call. +type ModerationSearchRepos_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Repos []*ModerationDefs_RepoView `json:"repos" cborgen:"repos"` +} + +// ModerationSearchRepos calls the XRPC method "tools.ozone.moderation.searchRepos". +// +// term: DEPRECATED: use 'q' instead +func ModerationSearchRepos(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, q string, term string) (*ModerationSearchRepos_Output, error) { + var out ModerationSearchRepos_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if q != "" { + params["q"] = q + } + if term != "" { + params["term"] = term + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.moderation.searchRepos", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/reportdefs.go b/api/ozone/reportdefs.go new file mode 100644 index 000000000..4faad21ed --- /dev/null +++ b/api/ozone/reportdefs.go @@ -0,0 +1,5 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.report.defs + +package ozone diff --git a/api/ozone/safelinkaddRule.go b/api/ozone/safelinkaddRule.go new file mode 100644 index 000000000..ecfb8138c --- /dev/null +++ b/api/ozone/safelinkaddRule.go @@ -0,0 +1,34 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.safelink.addRule + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SafelinkAddRule_Input is the input argument to a tools.ozone.safelink.addRule call. +type SafelinkAddRule_Input struct { + Action *string `json:"action" cborgen:"action"` + // comment: Optional comment about the decision + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // createdBy: Author DID. Only respected when using admin auth + CreatedBy *string `json:"createdBy,omitempty" cborgen:"createdBy,omitempty"` + Pattern *string `json:"pattern" cborgen:"pattern"` + Reason *string `json:"reason" cborgen:"reason"` + // url: The URL or domain to apply the rule to + Url string `json:"url" cborgen:"url"` +} + +// SafelinkAddRule calls the XRPC method "tools.ozone.safelink.addRule". +func SafelinkAddRule(ctx context.Context, c lexutil.LexClient, input *SafelinkAddRule_Input) (*SafelinkDefs_Event, error) { + var out SafelinkDefs_Event + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.safelink.addRule", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/safelinkdefs.go b/api/ozone/safelinkdefs.go new file mode 100644 index 000000000..d9aa66e87 --- /dev/null +++ b/api/ozone/safelinkdefs.go @@ -0,0 +1,43 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.safelink.defs + +package ozone + +// SafelinkDefs_Event is a "event" in the tools.ozone.safelink.defs schema. +// +// An event for URL safety decisions +type SafelinkDefs_Event struct { + Action *string `json:"action" cborgen:"action"` + // comment: Optional comment about the decision + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // createdBy: DID of the user who created this rule + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + EventType *string `json:"eventType" cborgen:"eventType"` + // id: Auto-incrementing row ID + Id int64 `json:"id" cborgen:"id"` + Pattern *string `json:"pattern" cborgen:"pattern"` + Reason *string `json:"reason" cborgen:"reason"` + // url: The URL that this rule applies to + Url string `json:"url" cborgen:"url"` +} + +// SafelinkDefs_UrlRule is a "urlRule" in the tools.ozone.safelink.defs schema. +// +// Input for creating a URL safety rule +type SafelinkDefs_UrlRule struct { + Action *string `json:"action" cborgen:"action"` + // comment: Optional comment about the decision + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // createdAt: Timestamp when the rule was created + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // createdBy: DID of the user added the rule. + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + Pattern *string `json:"pattern" cborgen:"pattern"` + Reason *string `json:"reason" cborgen:"reason"` + // updatedAt: Timestamp when the rule was last updated + UpdatedAt string `json:"updatedAt" cborgen:"updatedAt"` + // url: The URL or domain to apply the rule to + Url string `json:"url" cborgen:"url"` +} diff --git a/api/ozone/safelinkqueryEvents.go b/api/ozone/safelinkqueryEvents.go new file mode 100644 index 000000000..8fdd149d9 --- /dev/null +++ b/api/ozone/safelinkqueryEvents.go @@ -0,0 +1,42 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.safelink.queryEvents + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SafelinkQueryEvents_Input is the input argument to a tools.ozone.safelink.queryEvents call. +type SafelinkQueryEvents_Input struct { + // cursor: Cursor for pagination + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // limit: Maximum number of results to return + Limit *int64 `json:"limit,omitempty" cborgen:"limit,omitempty"` + // patternType: Filter by pattern type + PatternType *string `json:"patternType,omitempty" cborgen:"patternType,omitempty"` + // sortDirection: Sort direction + SortDirection *string `json:"sortDirection,omitempty" cborgen:"sortDirection,omitempty"` + // urls: Filter by specific URLs or domains + Urls []string `json:"urls,omitempty" cborgen:"urls,omitempty"` +} + +// SafelinkQueryEvents_Output is the output of a tools.ozone.safelink.queryEvents call. +type SafelinkQueryEvents_Output struct { + // cursor: Next cursor for pagination. Only present if there are more results. + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Events []*SafelinkDefs_Event `json:"events" cborgen:"events"` +} + +// SafelinkQueryEvents calls the XRPC method "tools.ozone.safelink.queryEvents". +func SafelinkQueryEvents(ctx context.Context, c lexutil.LexClient, input *SafelinkQueryEvents_Input) (*SafelinkQueryEvents_Output, error) { + var out SafelinkQueryEvents_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.safelink.queryEvents", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/safelinkqueryRules.go b/api/ozone/safelinkqueryRules.go new file mode 100644 index 000000000..9360b0e3f --- /dev/null +++ b/api/ozone/safelinkqueryRules.go @@ -0,0 +1,48 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.safelink.queryRules + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SafelinkQueryRules_Input is the input argument to a tools.ozone.safelink.queryRules call. +type SafelinkQueryRules_Input struct { + // actions: Filter by action types + Actions []string `json:"actions,omitempty" cborgen:"actions,omitempty"` + // createdBy: Filter by rule creator + CreatedBy *string `json:"createdBy,omitempty" cborgen:"createdBy,omitempty"` + // cursor: Cursor for pagination + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + // limit: Maximum number of results to return + Limit *int64 `json:"limit,omitempty" cborgen:"limit,omitempty"` + // patternType: Filter by pattern type + PatternType *string `json:"patternType,omitempty" cborgen:"patternType,omitempty"` + // reason: Filter by reason type + Reason *string `json:"reason,omitempty" cborgen:"reason,omitempty"` + // sortDirection: Sort direction + SortDirection *string `json:"sortDirection,omitempty" cborgen:"sortDirection,omitempty"` + // urls: Filter by specific URLs or domains + Urls []string `json:"urls,omitempty" cborgen:"urls,omitempty"` +} + +// SafelinkQueryRules_Output is the output of a tools.ozone.safelink.queryRules call. +type SafelinkQueryRules_Output struct { + // cursor: Next cursor for pagination. Only present if there are more results. + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Rules []*SafelinkDefs_UrlRule `json:"rules" cborgen:"rules"` +} + +// SafelinkQueryRules calls the XRPC method "tools.ozone.safelink.queryRules". +func SafelinkQueryRules(ctx context.Context, c lexutil.LexClient, input *SafelinkQueryRules_Input) (*SafelinkQueryRules_Output, error) { + var out SafelinkQueryRules_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.safelink.queryRules", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/safelinkremoveRule.go b/api/ozone/safelinkremoveRule.go new file mode 100644 index 000000000..92a13a432 --- /dev/null +++ b/api/ozone/safelinkremoveRule.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.safelink.removeRule + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SafelinkRemoveRule_Input is the input argument to a tools.ozone.safelink.removeRule call. +type SafelinkRemoveRule_Input struct { + // comment: Optional comment about why the rule is being removed + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // createdBy: Optional DID of the user. Only respected when using admin auth. + CreatedBy *string `json:"createdBy,omitempty" cborgen:"createdBy,omitempty"` + Pattern *string `json:"pattern" cborgen:"pattern"` + // url: The URL or domain to remove the rule for + Url string `json:"url" cborgen:"url"` +} + +// SafelinkRemoveRule calls the XRPC method "tools.ozone.safelink.removeRule". +func SafelinkRemoveRule(ctx context.Context, c lexutil.LexClient, input *SafelinkRemoveRule_Input) (*SafelinkDefs_Event, error) { + var out SafelinkDefs_Event + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.safelink.removeRule", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/safelinkupdateRule.go b/api/ozone/safelinkupdateRule.go new file mode 100644 index 000000000..ada1c7ce3 --- /dev/null +++ b/api/ozone/safelinkupdateRule.go @@ -0,0 +1,34 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.safelink.updateRule + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SafelinkUpdateRule_Input is the input argument to a tools.ozone.safelink.updateRule call. +type SafelinkUpdateRule_Input struct { + Action *string `json:"action" cborgen:"action"` + // comment: Optional comment about the update + Comment *string `json:"comment,omitempty" cborgen:"comment,omitempty"` + // createdBy: Optional DID to credit as the creator. Only respected for admin_token authentication. + CreatedBy *string `json:"createdBy,omitempty" cborgen:"createdBy,omitempty"` + Pattern *string `json:"pattern" cborgen:"pattern"` + Reason *string `json:"reason" cborgen:"reason"` + // url: The URL or domain to update the rule for + Url string `json:"url" cborgen:"url"` +} + +// SafelinkUpdateRule calls the XRPC method "tools.ozone.safelink.updateRule". +func SafelinkUpdateRule(ctx context.Context, c lexutil.LexClient, input *SafelinkUpdateRule_Input) (*SafelinkDefs_Event, error) { + var out SafelinkDefs_Event + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.safelink.updateRule", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/servergetConfig.go b/api/ozone/servergetConfig.go new file mode 100644 index 000000000..b6c03b2b4 --- /dev/null +++ b/api/ozone/servergetConfig.go @@ -0,0 +1,42 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.server.getConfig + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// ServerGetConfig_Output is the output of a tools.ozone.server.getConfig call. +type ServerGetConfig_Output struct { + Appview *ServerGetConfig_ServiceConfig `json:"appview,omitempty" cborgen:"appview,omitempty"` + BlobDivert *ServerGetConfig_ServiceConfig `json:"blobDivert,omitempty" cborgen:"blobDivert,omitempty"` + Chat *ServerGetConfig_ServiceConfig `json:"chat,omitempty" cborgen:"chat,omitempty"` + Pds *ServerGetConfig_ServiceConfig `json:"pds,omitempty" cborgen:"pds,omitempty"` + // verifierDid: The did of the verifier used for verification. + VerifierDid *string `json:"verifierDid,omitempty" cborgen:"verifierDid,omitempty"` + Viewer *ServerGetConfig_ViewerConfig `json:"viewer,omitempty" cborgen:"viewer,omitempty"` +} + +// ServerGetConfig_ServiceConfig is a "serviceConfig" in the tools.ozone.server.getConfig schema. +type ServerGetConfig_ServiceConfig struct { + Url *string `json:"url,omitempty" cborgen:"url,omitempty"` +} + +// ServerGetConfig_ViewerConfig is a "viewerConfig" in the tools.ozone.server.getConfig schema. +type ServerGetConfig_ViewerConfig struct { + Role *string `json:"role,omitempty" cborgen:"role,omitempty"` +} + +// ServerGetConfig calls the XRPC method "tools.ozone.server.getConfig". +func ServerGetConfig(ctx context.Context, c lexutil.LexClient) (*ServerGetConfig_Output, error) { + var out ServerGetConfig_Output + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.server.getConfig", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/setaddValues.go b/api/ozone/setaddValues.go new file mode 100644 index 000000000..2b59ea0b3 --- /dev/null +++ b/api/ozone/setaddValues.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.set.addValues + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SetAddValues_Input is the input argument to a tools.ozone.set.addValues call. +type SetAddValues_Input struct { + // name: Name of the set to add values to + Name string `json:"name" cborgen:"name"` + // values: Array of string values to add to the set + Values []string `json:"values" cborgen:"values"` +} + +// SetAddValues calls the XRPC method "tools.ozone.set.addValues". +func SetAddValues(ctx context.Context, c lexutil.LexClient, input *SetAddValues_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.set.addValues", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/ozone/setdefs.go b/api/ozone/setdefs.go new file mode 100644 index 000000000..3d89bdb67 --- /dev/null +++ b/api/ozone/setdefs.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.set.defs + +package ozone + +// SetDefs_Set is a "set" in the tools.ozone.set.defs schema. +type SetDefs_Set struct { + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + Name string `json:"name" cborgen:"name"` +} + +// SetDefs_SetView is a "setView" in the tools.ozone.set.defs schema. +type SetDefs_SetView struct { + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + Name string `json:"name" cborgen:"name"` + SetSize int64 `json:"setSize" cborgen:"setSize"` + UpdatedAt string `json:"updatedAt" cborgen:"updatedAt"` +} diff --git a/api/ozone/setdeleteSet.go b/api/ozone/setdeleteSet.go new file mode 100644 index 000000000..d48f6e507 --- /dev/null +++ b/api/ozone/setdeleteSet.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.set.deleteSet + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SetDeleteSet_Input is the input argument to a tools.ozone.set.deleteSet call. +type SetDeleteSet_Input struct { + // name: Name of the set to delete + Name string `json:"name" cborgen:"name"` +} + +// SetDeleteSet_Output is the output of a tools.ozone.set.deleteSet call. +type SetDeleteSet_Output struct { +} + +// SetDeleteSet calls the XRPC method "tools.ozone.set.deleteSet". +func SetDeleteSet(ctx context.Context, c lexutil.LexClient, input *SetDeleteSet_Input) (*SetDeleteSet_Output, error) { + var out SetDeleteSet_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.set.deleteSet", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/setdeleteValues.go b/api/ozone/setdeleteValues.go new file mode 100644 index 000000000..96cc1f355 --- /dev/null +++ b/api/ozone/setdeleteValues.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.set.deleteValues + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SetDeleteValues_Input is the input argument to a tools.ozone.set.deleteValues call. +type SetDeleteValues_Input struct { + // name: Name of the set to delete values from + Name string `json:"name" cborgen:"name"` + // values: Array of string values to delete from the set + Values []string `json:"values" cborgen:"values"` +} + +// SetDeleteValues calls the XRPC method "tools.ozone.set.deleteValues". +func SetDeleteValues(ctx context.Context, c lexutil.LexClient, input *SetDeleteValues_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.set.deleteValues", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/ozone/setgetValues.go b/api/ozone/setgetValues.go new file mode 100644 index 000000000..eb5bcc2e8 --- /dev/null +++ b/api/ozone/setgetValues.go @@ -0,0 +1,37 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.set.getValues + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SetGetValues_Output is the output of a tools.ozone.set.getValues call. +type SetGetValues_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Set *SetDefs_SetView `json:"set" cborgen:"set"` + Values []string `json:"values" cborgen:"values"` +} + +// SetGetValues calls the XRPC method "tools.ozone.set.getValues". +func SetGetValues(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, name string) (*SetGetValues_Output, error) { + var out SetGetValues_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["name"] = name + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.set.getValues", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/setquerySets.go b/api/ozone/setquerySets.go new file mode 100644 index 000000000..d853cc3ee --- /dev/null +++ b/api/ozone/setquerySets.go @@ -0,0 +1,46 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.set.querySets + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SetQuerySets_Output is the output of a tools.ozone.set.querySets call. +type SetQuerySets_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Sets []*SetDefs_SetView `json:"sets" cborgen:"sets"` +} + +// SetQuerySets calls the XRPC method "tools.ozone.set.querySets". +// +// sortDirection: Defaults to ascending order of name field. +func SetQuerySets(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, namePrefix string, sortBy string, sortDirection string) (*SetQuerySets_Output, error) { + var out SetQuerySets_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + if namePrefix != "" { + params["namePrefix"] = namePrefix + } + if sortBy != "" { + params["sortBy"] = sortBy + } + if sortDirection != "" { + params["sortDirection"] = sortDirection + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.set.querySets", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/settingdefs.go b/api/ozone/settingdefs.go new file mode 100644 index 000000000..f0b4e5d44 --- /dev/null +++ b/api/ozone/settingdefs.go @@ -0,0 +1,23 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.setting.defs + +package ozone + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SettingDefs_Option is a "option" in the tools.ozone.setting.defs schema. +type SettingDefs_Option struct { + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + CreatedBy string `json:"createdBy" cborgen:"createdBy"` + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + Did string `json:"did" cborgen:"did"` + Key string `json:"key" cborgen:"key"` + LastUpdatedBy string `json:"lastUpdatedBy" cborgen:"lastUpdatedBy"` + ManagerRole *string `json:"managerRole,omitempty" cborgen:"managerRole,omitempty"` + Scope string `json:"scope" cborgen:"scope"` + UpdatedAt *string `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"` + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` +} diff --git a/api/ozone/settinglistOptions.go b/api/ozone/settinglistOptions.go new file mode 100644 index 000000000..07e5801ec --- /dev/null +++ b/api/ozone/settinglistOptions.go @@ -0,0 +1,47 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.setting.listOptions + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SettingListOptions_Output is the output of a tools.ozone.setting.listOptions call. +type SettingListOptions_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Options []*SettingDefs_Option `json:"options" cborgen:"options"` +} + +// SettingListOptions calls the XRPC method "tools.ozone.setting.listOptions". +// +// keys: Filter for only the specified keys. Ignored if prefix is provided +// prefix: Filter keys by prefix +func SettingListOptions(ctx context.Context, c lexutil.LexClient, cursor string, keys []string, limit int64, prefix string, scope string) (*SettingListOptions_Output, error) { + var out SettingListOptions_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if len(keys) != 0 { + params["keys"] = keys + } + if limit != 0 { + params["limit"] = limit + } + if prefix != "" { + params["prefix"] = prefix + } + if scope != "" { + params["scope"] = scope + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.setting.listOptions", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/settingremoveOptions.go b/api/ozone/settingremoveOptions.go new file mode 100644 index 000000000..81e6ff13c --- /dev/null +++ b/api/ozone/settingremoveOptions.go @@ -0,0 +1,31 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.setting.removeOptions + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SettingRemoveOptions_Input is the input argument to a tools.ozone.setting.removeOptions call. +type SettingRemoveOptions_Input struct { + Keys []string `json:"keys" cborgen:"keys"` + Scope string `json:"scope" cborgen:"scope"` +} + +// SettingRemoveOptions_Output is the output of a tools.ozone.setting.removeOptions call. +type SettingRemoveOptions_Output struct { +} + +// SettingRemoveOptions calls the XRPC method "tools.ozone.setting.removeOptions". +func SettingRemoveOptions(ctx context.Context, c lexutil.LexClient, input *SettingRemoveOptions_Input) (*SettingRemoveOptions_Output, error) { + var out SettingRemoveOptions_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.setting.removeOptions", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/settingupsertOption.go b/api/ozone/settingupsertOption.go new file mode 100644 index 000000000..8d12dac92 --- /dev/null +++ b/api/ozone/settingupsertOption.go @@ -0,0 +1,35 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.setting.upsertOption + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SettingUpsertOption_Input is the input argument to a tools.ozone.setting.upsertOption call. +type SettingUpsertOption_Input struct { + Description *string `json:"description,omitempty" cborgen:"description,omitempty"` + Key string `json:"key" cborgen:"key"` + ManagerRole *string `json:"managerRole,omitempty" cborgen:"managerRole,omitempty"` + Scope string `json:"scope" cborgen:"scope"` + Value *lexutil.LexiconTypeDecoder `json:"value" cborgen:"value"` +} + +// SettingUpsertOption_Output is the output of a tools.ozone.setting.upsertOption call. +type SettingUpsertOption_Output struct { + Option *SettingDefs_Option `json:"option" cborgen:"option"` +} + +// SettingUpsertOption calls the XRPC method "tools.ozone.setting.upsertOption". +func SettingUpsertOption(ctx context.Context, c lexutil.LexClient, input *SettingUpsertOption_Input) (*SettingUpsertOption_Output, error) { + var out SettingUpsertOption_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.setting.upsertOption", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/setupsertSet.go b/api/ozone/setupsertSet.go new file mode 100644 index 000000000..ccf565ec3 --- /dev/null +++ b/api/ozone/setupsertSet.go @@ -0,0 +1,21 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.set.upsertSet + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SetUpsertSet calls the XRPC method "tools.ozone.set.upsertSet". +func SetUpsertSet(ctx context.Context, c lexutil.LexClient, input *SetDefs_Set) (*SetDefs_SetView, error) { + var out SetDefs_SetView + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.set.upsertSet", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/signaturedefs.go b/api/ozone/signaturedefs.go new file mode 100644 index 000000000..a9374b224 --- /dev/null +++ b/api/ozone/signaturedefs.go @@ -0,0 +1,11 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.signature.defs + +package ozone + +// SignatureDefs_SigDetail is a "sigDetail" in the tools.ozone.signature.defs schema. +type SignatureDefs_SigDetail struct { + Property string `json:"property" cborgen:"property"` + Value string `json:"value" cborgen:"value"` +} diff --git a/api/ozone/signaturefindCorrelation.go b/api/ozone/signaturefindCorrelation.go new file mode 100644 index 000000000..9d28ed684 --- /dev/null +++ b/api/ozone/signaturefindCorrelation.go @@ -0,0 +1,29 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.signature.findCorrelation + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SignatureFindCorrelation_Output is the output of a tools.ozone.signature.findCorrelation call. +type SignatureFindCorrelation_Output struct { + Details []*SignatureDefs_SigDetail `json:"details" cborgen:"details"` +} + +// SignatureFindCorrelation calls the XRPC method "tools.ozone.signature.findCorrelation". +func SignatureFindCorrelation(ctx context.Context, c lexutil.LexClient, dids []string) (*SignatureFindCorrelation_Output, error) { + var out SignatureFindCorrelation_Output + + params := map[string]interface{}{} + params["dids"] = dids + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.signature.findCorrelation", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/signaturefindRelatedAccounts.go b/api/ozone/signaturefindRelatedAccounts.go new file mode 100644 index 000000000..7a2e348f8 --- /dev/null +++ b/api/ozone/signaturefindRelatedAccounts.go @@ -0,0 +1,43 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.signature.findRelatedAccounts + +package ozone + +import ( + "context" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SignatureFindRelatedAccounts_Output is the output of a tools.ozone.signature.findRelatedAccounts call. +type SignatureFindRelatedAccounts_Output struct { + Accounts []*SignatureFindRelatedAccounts_RelatedAccount `json:"accounts" cborgen:"accounts"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// SignatureFindRelatedAccounts_RelatedAccount is a "relatedAccount" in the tools.ozone.signature.findRelatedAccounts schema. +type SignatureFindRelatedAccounts_RelatedAccount struct { + Account *comatproto.AdminDefs_AccountView `json:"account" cborgen:"account"` + Similarities []*SignatureDefs_SigDetail `json:"similarities,omitempty" cborgen:"similarities,omitempty"` +} + +// SignatureFindRelatedAccounts calls the XRPC method "tools.ozone.signature.findRelatedAccounts". +func SignatureFindRelatedAccounts(ctx context.Context, c lexutil.LexClient, cursor string, did string, limit int64) (*SignatureFindRelatedAccounts_Output, error) { + var out SignatureFindRelatedAccounts_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + params["did"] = did + if limit != 0 { + params["limit"] = limit + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.signature.findRelatedAccounts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/signaturesearchAccounts.go b/api/ozone/signaturesearchAccounts.go new file mode 100644 index 000000000..f41bbae43 --- /dev/null +++ b/api/ozone/signaturesearchAccounts.go @@ -0,0 +1,37 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.signature.searchAccounts + +package ozone + +import ( + "context" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// SignatureSearchAccounts_Output is the output of a tools.ozone.signature.searchAccounts call. +type SignatureSearchAccounts_Output struct { + Accounts []*comatproto.AdminDefs_AccountView `json:"accounts" cborgen:"accounts"` + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` +} + +// SignatureSearchAccounts calls the XRPC method "tools.ozone.signature.searchAccounts". +func SignatureSearchAccounts(ctx context.Context, c lexutil.LexClient, cursor string, limit int64, values []string) (*SignatureSearchAccounts_Output, error) { + var out SignatureSearchAccounts_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if limit != 0 { + params["limit"] = limit + } + params["values"] = values + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.signature.searchAccounts", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/teamaddMember.go b/api/ozone/teamaddMember.go new file mode 100644 index 000000000..bf0e0b68b --- /dev/null +++ b/api/ozone/teamaddMember.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.team.addMember + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TeamAddMember_Input is the input argument to a tools.ozone.team.addMember call. +type TeamAddMember_Input struct { + Did string `json:"did" cborgen:"did"` + Role string `json:"role" cborgen:"role"` +} + +// TeamAddMember calls the XRPC method "tools.ozone.team.addMember". +func TeamAddMember(ctx context.Context, c lexutil.LexClient, input *TeamAddMember_Input) (*TeamDefs_Member, error) { + var out TeamDefs_Member + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.team.addMember", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/teamdefs.go b/api/ozone/teamdefs.go new file mode 100644 index 000000000..5b5f2c092 --- /dev/null +++ b/api/ozone/teamdefs.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.team.defs + +package ozone + +import ( + appbsky "github.com/bluesky-social/indigo/api/bsky" +) + +// TeamDefs_Member is a "member" in the tools.ozone.team.defs schema. +type TeamDefs_Member struct { + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + Did string `json:"did" cborgen:"did"` + Disabled *bool `json:"disabled,omitempty" cborgen:"disabled,omitempty"` + LastUpdatedBy *string `json:"lastUpdatedBy,omitempty" cborgen:"lastUpdatedBy,omitempty"` + Profile *appbsky.ActorDefs_ProfileViewDetailed `json:"profile,omitempty" cborgen:"profile,omitempty"` + Role string `json:"role" cborgen:"role"` + UpdatedAt *string `json:"updatedAt,omitempty" cborgen:"updatedAt,omitempty"` +} diff --git a/api/ozone/teamdeleteMember.go b/api/ozone/teamdeleteMember.go new file mode 100644 index 000000000..d8ed32115 --- /dev/null +++ b/api/ozone/teamdeleteMember.go @@ -0,0 +1,25 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.team.deleteMember + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TeamDeleteMember_Input is the input argument to a tools.ozone.team.deleteMember call. +type TeamDeleteMember_Input struct { + Did string `json:"did" cborgen:"did"` +} + +// TeamDeleteMember calls the XRPC method "tools.ozone.team.deleteMember". +func TeamDeleteMember(ctx context.Context, c lexutil.LexClient, input *TeamDeleteMember_Input) error { + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.team.deleteMember", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/ozone/teamlistMembers.go b/api/ozone/teamlistMembers.go new file mode 100644 index 000000000..7e04bd432 --- /dev/null +++ b/api/ozone/teamlistMembers.go @@ -0,0 +1,44 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.team.listMembers + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TeamListMembers_Output is the output of a tools.ozone.team.listMembers call. +type TeamListMembers_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Members []*TeamDefs_Member `json:"members" cborgen:"members"` +} + +// TeamListMembers calls the XRPC method "tools.ozone.team.listMembers". +func TeamListMembers(ctx context.Context, c lexutil.LexClient, cursor string, disabled bool, limit int64, q string, roles []string) (*TeamListMembers_Output, error) { + var out TeamListMembers_Output + + params := map[string]interface{}{} + if cursor != "" { + params["cursor"] = cursor + } + if disabled { + params["disabled"] = disabled + } + if limit != 0 { + params["limit"] = limit + } + if q != "" { + params["q"] = q + } + if len(roles) != 0 { + params["roles"] = roles + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.team.listMembers", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/teamupdateMember.go b/api/ozone/teamupdateMember.go new file mode 100644 index 000000000..1672cfc7f --- /dev/null +++ b/api/ozone/teamupdateMember.go @@ -0,0 +1,28 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.team.updateMember + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// TeamUpdateMember_Input is the input argument to a tools.ozone.team.updateMember call. +type TeamUpdateMember_Input struct { + Did string `json:"did" cborgen:"did"` + Disabled *bool `json:"disabled,omitempty" cborgen:"disabled,omitempty"` + Role *string `json:"role,omitempty" cborgen:"role,omitempty"` +} + +// TeamUpdateMember calls the XRPC method "tools.ozone.team.updateMember". +func TeamUpdateMember(ctx context.Context, c lexutil.LexClient, input *TeamUpdateMember_Input) (*TeamDefs_Member, error) { + var out TeamDefs_Member + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.team.updateMember", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/verificationdefs.go b/api/ozone/verificationdefs.go new file mode 100644 index 000000000..b4154173c --- /dev/null +++ b/api/ozone/verificationdefs.go @@ -0,0 +1,110 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.verification.defs + +package ozone + +import ( + "encoding/json" + "fmt" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// VerificationDefs_VerificationView is a "verificationView" in the tools.ozone.verification.defs schema. +// +// Verification data for the associated subject. +type VerificationDefs_VerificationView struct { + // createdAt: Timestamp when the verification was created. + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + // displayName: Display name of the subject the verification applies to at the moment of verifying, which might not be the same at the time of viewing. The verification is only valid if the current displayName matches the one at the time of verifying. + DisplayName string `json:"displayName" cborgen:"displayName"` + // handle: Handle of the subject the verification applies to at the moment of verifying, which might not be the same at the time of viewing. The verification is only valid if the current handle matches the one at the time of verifying. + Handle string `json:"handle" cborgen:"handle"` + // issuer: The user who issued this verification. + Issuer string `json:"issuer" cborgen:"issuer"` + IssuerProfile *lexutil.LexiconTypeDecoder `json:"issuerProfile,omitempty" cborgen:"issuerProfile,omitempty"` + IssuerRepo *VerificationDefs_VerificationView_IssuerRepo `json:"issuerRepo,omitempty" cborgen:"issuerRepo,omitempty"` + // revokeReason: Describes the reason for revocation, also indicating that the verification is no longer valid. + RevokeReason *string `json:"revokeReason,omitempty" cborgen:"revokeReason,omitempty"` + // revokedAt: Timestamp when the verification was revoked. + RevokedAt *string `json:"revokedAt,omitempty" cborgen:"revokedAt,omitempty"` + // revokedBy: The user who revoked this verification. + RevokedBy *string `json:"revokedBy,omitempty" cborgen:"revokedBy,omitempty"` + // subject: The subject of the verification. + Subject string `json:"subject" cborgen:"subject"` + SubjectProfile *lexutil.LexiconTypeDecoder `json:"subjectProfile,omitempty" cborgen:"subjectProfile,omitempty"` + SubjectRepo *VerificationDefs_VerificationView_SubjectRepo `json:"subjectRepo,omitempty" cborgen:"subjectRepo,omitempty"` + // uri: The AT-URI of the verification record. + Uri string `json:"uri" cborgen:"uri"` +} + +type VerificationDefs_VerificationView_IssuerRepo struct { + ModerationDefs_RepoViewDetail *ModerationDefs_RepoViewDetail + ModerationDefs_RepoViewNotFound *ModerationDefs_RepoViewNotFound +} + +func (t *VerificationDefs_VerificationView_IssuerRepo) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_RepoViewDetail != nil { + t.ModerationDefs_RepoViewDetail.LexiconTypeID = "tools.ozone.moderation.defs#repoViewDetail" + return json.Marshal(t.ModerationDefs_RepoViewDetail) + } + if t.ModerationDefs_RepoViewNotFound != nil { + t.ModerationDefs_RepoViewNotFound.LexiconTypeID = "tools.ozone.moderation.defs#repoViewNotFound" + return json.Marshal(t.ModerationDefs_RepoViewNotFound) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *VerificationDefs_VerificationView_IssuerRepo) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#repoViewDetail": + t.ModerationDefs_RepoViewDetail = new(ModerationDefs_RepoViewDetail) + return json.Unmarshal(b, t.ModerationDefs_RepoViewDetail) + case "tools.ozone.moderation.defs#repoViewNotFound": + t.ModerationDefs_RepoViewNotFound = new(ModerationDefs_RepoViewNotFound) + return json.Unmarshal(b, t.ModerationDefs_RepoViewNotFound) + default: + return nil + } +} + +type VerificationDefs_VerificationView_SubjectRepo struct { + ModerationDefs_RepoViewDetail *ModerationDefs_RepoViewDetail + ModerationDefs_RepoViewNotFound *ModerationDefs_RepoViewNotFound +} + +func (t *VerificationDefs_VerificationView_SubjectRepo) MarshalJSON() ([]byte, error) { + if t.ModerationDefs_RepoViewDetail != nil { + t.ModerationDefs_RepoViewDetail.LexiconTypeID = "tools.ozone.moderation.defs#repoViewDetail" + return json.Marshal(t.ModerationDefs_RepoViewDetail) + } + if t.ModerationDefs_RepoViewNotFound != nil { + t.ModerationDefs_RepoViewNotFound.LexiconTypeID = "tools.ozone.moderation.defs#repoViewNotFound" + return json.Marshal(t.ModerationDefs_RepoViewNotFound) + } + return nil, fmt.Errorf("can not marshal empty union as JSON") +} + +func (t *VerificationDefs_VerificationView_SubjectRepo) UnmarshalJSON(b []byte) error { + typ, err := lexutil.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "tools.ozone.moderation.defs#repoViewDetail": + t.ModerationDefs_RepoViewDetail = new(ModerationDefs_RepoViewDetail) + return json.Unmarshal(b, t.ModerationDefs_RepoViewDetail) + case "tools.ozone.moderation.defs#repoViewNotFound": + t.ModerationDefs_RepoViewNotFound = new(ModerationDefs_RepoViewNotFound) + return json.Unmarshal(b, t.ModerationDefs_RepoViewNotFound) + default: + return nil + } +} diff --git a/api/ozone/verificationgrantVerifications.go b/api/ozone/verificationgrantVerifications.go new file mode 100644 index 000000000..fc4fa67c5 --- /dev/null +++ b/api/ozone/verificationgrantVerifications.go @@ -0,0 +1,55 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.verification.grantVerifications + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// VerificationGrantVerifications_GrantError is a "grantError" in the tools.ozone.verification.grantVerifications schema. +// +// Error object for failed verifications. +type VerificationGrantVerifications_GrantError struct { + // error: Error message describing the reason for failure. + Error string `json:"error" cborgen:"error"` + // subject: The did of the subject being verified + Subject string `json:"subject" cborgen:"subject"` +} + +// VerificationGrantVerifications_Input is the input argument to a tools.ozone.verification.grantVerifications call. +type VerificationGrantVerifications_Input struct { + // verifications: Array of verification requests to process + Verifications []*VerificationGrantVerifications_VerificationInput `json:"verifications" cborgen:"verifications"` +} + +// VerificationGrantVerifications_Output is the output of a tools.ozone.verification.grantVerifications call. +type VerificationGrantVerifications_Output struct { + FailedVerifications []*VerificationGrantVerifications_GrantError `json:"failedVerifications" cborgen:"failedVerifications"` + Verifications []*VerificationDefs_VerificationView `json:"verifications" cborgen:"verifications"` +} + +// VerificationGrantVerifications_VerificationInput is the input argument to a tools.ozone.verification.grantVerifications call. +type VerificationGrantVerifications_VerificationInput struct { + // createdAt: Timestamp for verification record. Defaults to current time when not specified. + CreatedAt *string `json:"createdAt,omitempty" cborgen:"createdAt,omitempty"` + // displayName: Display name of the subject the verification applies to at the moment of verifying. + DisplayName string `json:"displayName" cborgen:"displayName"` + // handle: Handle of the subject the verification applies to at the moment of verifying. + Handle string `json:"handle" cborgen:"handle"` + // subject: The did of the subject being verified + Subject string `json:"subject" cborgen:"subject"` +} + +// VerificationGrantVerifications calls the XRPC method "tools.ozone.verification.grantVerifications". +func VerificationGrantVerifications(ctx context.Context, c lexutil.LexClient, input *VerificationGrantVerifications_Input) (*VerificationGrantVerifications_Output, error) { + var out VerificationGrantVerifications_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.verification.grantVerifications", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/verificationlistVerifications.go b/api/ozone/verificationlistVerifications.go new file mode 100644 index 000000000..ea8d79095 --- /dev/null +++ b/api/ozone/verificationlistVerifications.go @@ -0,0 +1,62 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.verification.listVerifications + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// VerificationListVerifications_Output is the output of a tools.ozone.verification.listVerifications call. +type VerificationListVerifications_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Verifications []*VerificationDefs_VerificationView `json:"verifications" cborgen:"verifications"` +} + +// VerificationListVerifications calls the XRPC method "tools.ozone.verification.listVerifications". +// +// createdAfter: Filter to verifications created after this timestamp +// createdBefore: Filter to verifications created before this timestamp +// cursor: Pagination cursor +// isRevoked: Filter to verifications that are revoked or not. By default, includes both. +// issuers: Filter to verifications from specific issuers +// limit: Maximum number of results to return +// sortDirection: Sort direction for creation date +// subjects: Filter to specific verified DIDs +func VerificationListVerifications(ctx context.Context, c lexutil.LexClient, createdAfter string, createdBefore string, cursor string, isRevoked bool, issuers []string, limit int64, sortDirection string, subjects []string) (*VerificationListVerifications_Output, error) { + var out VerificationListVerifications_Output + + params := map[string]interface{}{} + if createdAfter != "" { + params["createdAfter"] = createdAfter + } + if createdBefore != "" { + params["createdBefore"] = createdBefore + } + if cursor != "" { + params["cursor"] = cursor + } + if isRevoked { + params["isRevoked"] = isRevoked + } + if len(issuers) != 0 { + params["issuers"] = issuers + } + if limit != 0 { + params["limit"] = limit + } + if sortDirection != "" { + params["sortDirection"] = sortDirection + } + if len(subjects) != 0 { + params["subjects"] = subjects + } + if err := c.LexDo(ctx, lexutil.Query, "", "tools.ozone.verification.listVerifications", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/ozone/verificationrevokeVerifications.go b/api/ozone/verificationrevokeVerifications.go new file mode 100644 index 000000000..2ddcd17cc --- /dev/null +++ b/api/ozone/verificationrevokeVerifications.go @@ -0,0 +1,47 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +// Lexicon schema: tools.ozone.verification.revokeVerifications + +package ozone + +import ( + "context" + + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// VerificationRevokeVerifications_Input is the input argument to a tools.ozone.verification.revokeVerifications call. +type VerificationRevokeVerifications_Input struct { + // revokeReason: Reason for revoking the verification. This is optional and can be omitted if not needed. + RevokeReason *string `json:"revokeReason,omitempty" cborgen:"revokeReason,omitempty"` + // uris: Array of verification record uris to revoke + Uris []string `json:"uris" cborgen:"uris"` +} + +// VerificationRevokeVerifications_Output is the output of a tools.ozone.verification.revokeVerifications call. +type VerificationRevokeVerifications_Output struct { + // failedRevocations: List of verification uris that couldn't be revoked, including failure reasons + FailedRevocations []*VerificationRevokeVerifications_RevokeError `json:"failedRevocations" cborgen:"failedRevocations"` + // revokedVerifications: List of verification uris successfully revoked + RevokedVerifications []string `json:"revokedVerifications" cborgen:"revokedVerifications"` +} + +// VerificationRevokeVerifications_RevokeError is a "revokeError" in the tools.ozone.verification.revokeVerifications schema. +// +// Error object for failed revocations +type VerificationRevokeVerifications_RevokeError struct { + // error: Description of the error that occurred during revocation. + Error string `json:"error" cborgen:"error"` + // uri: The AT-URI of the verification record that failed to revoke. + Uri string `json:"uri" cborgen:"uri"` +} + +// VerificationRevokeVerifications calls the XRPC method "tools.ozone.verification.revokeVerifications". +func VerificationRevokeVerifications(ctx context.Context, c lexutil.LexClient, input *VerificationRevokeVerifications_Input) (*VerificationRevokeVerifications_Output, error) { + var out VerificationRevokeVerifications_Output + if err := c.LexDo(ctx, lexutil.Procedure, "application/json", "tools.ozone.verification.revokeVerifications", nil, input, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/atproto/atclient/admin_auth.go b/atproto/atclient/admin_auth.go new file mode 100644 index 000000000..b80ed0785 --- /dev/null +++ b/atproto/atclient/admin_auth.go @@ -0,0 +1,23 @@ +package atclient + +import ( + "net/http" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Simple [AuthMethod] implementation for atproto "admin auth". +type AdminAuth struct { + Password string +} + +func (a *AdminAuth) DoWithAuth(c *http.Client, req *http.Request, endpoint syntax.NSID) (*http.Response, error) { + req.SetBasicAuth("admin", a.Password) + return c.Do(req) +} + +func NewAdminClient(host, password string) *APIClient { + c := NewAPIClient(host) + c.Auth = &AdminAuth{Password: password} + return c +} diff --git a/atproto/atclient/admin_auth_test.go b/atproto/atclient/admin_auth_test.go new file mode 100644 index 000000000..19bb043eb --- /dev/null +++ b/atproto/atclient/admin_auth_test.go @@ -0,0 +1,51 @@ +package atclient + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" +) + +func adminHandler(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + if ok && username == "admin" && password == "secret" { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintln(w, "{\"status\":\"success\"}") + return + } + w.Header().Set("WWW-Authenticate", `Basic realm="admin", charset="UTF-8"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) +} + +func TestAdminAuth(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + var apierr *APIError + + srv := httptest.NewServer(http.HandlerFunc(adminHandler)) + defer srv.Close() + + { + c := NewAPIClient(srv.URL) + err := c.Get(ctx, syntax.NSID("com.example.get"), nil, nil) + assert.ErrorAs(err, &apierr) + } + + { + c := NewAdminClient(srv.URL, "wrong") + err := c.Get(ctx, syntax.NSID("com.example.get"), nil, nil) + assert.ErrorAs(err, &apierr) + } + + { + c := NewAdminClient(srv.URL, "secret") + err := c.Get(ctx, syntax.NSID("com.example.get"), nil, nil) + assert.NoError(err) + } +} diff --git a/atproto/atclient/apiclient.go b/atproto/atclient/apiclient.go new file mode 100644 index 000000000..0a0fe732f --- /dev/null +++ b/atproto/atclient/apiclient.go @@ -0,0 +1,200 @@ +package atclient + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Interface for auth implementations which can be used with [APIClient]. +type AuthMethod interface { + // Endpoint parameter is included for auth methods which need to include the NSID in authorization tokens + DoWithAuth(c *http.Client, req *http.Request, endpoint syntax.NSID) (*http.Response, error) +} + +// General purpose client for atproto "XRPC" API endpoints. +type APIClient struct { + // Inner HTTP client. May be customized after the overall [APIClient] struct is created; for example to set a default request timeout. + Client *http.Client + + // Host URL prefix: scheme, hostname, and port. This field is required. + Host string + + // Optional auth client "middleware". + Auth AuthMethod + + // Optional HTTP headers which will be included in all requests. Only a single value per key is included; request-level headers will override any client-level defaults. + Headers http.Header + + // optional authenticated account DID for this client. Does not change client behavior; this field is included as a convenience for calling code, logging, etc. + AccountDID *syntax.DID +} + +// Creates a simple APIClient for the provided host. This is appropriate for use with unauthenticated ("public") atproto API endpoints, or to use as a base client to add authentication. +// +// Uses [http.DefaultClient], and sets a default User-Agent. +func NewAPIClient(host string) *APIClient { + return &APIClient{ + Client: http.DefaultClient, + Host: host, + Headers: map[string][]string{ + "User-Agent": []string{"indigo-sdk"}, + }, + } +} + +// High-level helper for simple JSON "Query" API calls. +// +// This method automatically parses non-successful responses to [APIError]. +// +// For Query endpoints which return non-JSON data, or other situations needing complete configuration of the request and response, use the [APIClient.Do] method. +func (c *APIClient) Get(ctx context.Context, endpoint syntax.NSID, params map[string]any, out any) error { + + req := NewAPIRequest(http.MethodGet, endpoint, nil) + req.Headers.Set("Accept", "application/json") + + if params != nil { + qp, err := ParseParams(params) + if err != nil { + return err + } + req.QueryParams = qp + } + + resp, err := c.Do(ctx, req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + var eb ErrorBody + if err := json.NewDecoder(resp.Body).Decode(&eb); err != nil { + return &APIError{StatusCode: resp.StatusCode} + } + return eb.APIError(resp.StatusCode) + } + + if out == nil { + // drain body before returning + io.ReadAll(resp.Body) + return nil + } + + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("failed decoding JSON response body: %w", err) + } + return nil +} + +// High-level helper for simple JSON-to-JSON "Procedure" API calls, with no query params. +// +// This method automatically parses non-successful responses to [APIError]. +// +// For Query endpoints which expect non-JSON request bodies; return non-JSON responses; direct use of [io.Reader] for the request body; or other situations needing complete configuration of the request and response, use the [APIClient.Do] method. +func (c *APIClient) Post(ctx context.Context, endpoint syntax.NSID, body any, out any) error { + bodyJSON, err := json.Marshal(body) + if err != nil { + return err + } + + req := NewAPIRequest(http.MethodPost, endpoint, bytes.NewReader(bodyJSON)) + req.Headers.Set("Accept", "application/json") + req.Headers.Set("Content-Type", "application/json") + + resp, err := c.Do(ctx, req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + var eb ErrorBody + if err := json.NewDecoder(resp.Body).Decode(&eb); err != nil { + return &APIError{StatusCode: resp.StatusCode} + } + return eb.APIError(resp.StatusCode) + } + + if out == nil { + // drain body before returning + io.ReadAll(resp.Body) + return nil + } + + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("failed decoding JSON response body: %w", err) + } + return nil +} + +// Full-featured method for atproto API requests. +// +// TODO: this does not currently parse API error response JSON body to [APIError], thought it might in the future. +func (c *APIClient) Do(ctx context.Context, req *APIRequest) (*http.Response, error) { + + if c.Client == nil { + c.Client = http.DefaultClient + } + + httpReq, err := req.HTTPRequest(ctx, c.Host, c.Headers) + if err != nil { + return nil, err + } + + var resp *http.Response + if c.Auth != nil { + resp, err = c.Auth.DoWithAuth(c.Client, httpReq, req.Endpoint) + } else { + resp, err = c.Client.Do(httpReq) + } + if err != nil { + return nil, err + } + return resp, nil +} + +// Returns a shallow copy of the APIClient with the provided service ref configured as a proxy header. +// +// To configure service proxying without creating a copy, simply set the 'Atproto-Proxy' header. +func (c *APIClient) WithService(ref string) *APIClient { + hdr := c.Headers.Clone() + hdr.Set("Atproto-Proxy", ref) + out := APIClient{ + Client: c.Client, + Host: c.Host, + Auth: c.Auth, + Headers: hdr, + AccountDID: c.AccountDID, + } + return &out +} + +// Configures labeler header ('Atproto-Accept-Labelers') with the indicated "redact" level labelers, and regular labelers. +// +// Overwrites any existing client-level header value. +func (c *APIClient) SetLabelers(redact, other []syntax.DID) { + c.Headers.Set("Atproto-Accept-Labelers", encodeLabelerHeader(redact, other)) +} + +func encodeLabelerHeader(redact, other []syntax.DID) string { + val := "" + for _, did := range redact { + if val != "" { + val = val + "," + } + val = fmt.Sprintf("%s%s;redact", val, did.String()) + } + for _, did := range other { + if val != "" { + val = val + "," + } + val = val + did.String() + } + return val +} diff --git a/atproto/atclient/apiclient_test.go b/atproto/atclient/apiclient_test.go new file mode 100644 index 000000000..b227f5dc3 --- /dev/null +++ b/atproto/atclient/apiclient_test.go @@ -0,0 +1,21 @@ +package atclient + +import ( + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" +) + +func TestEncodeLabelerHeader(t *testing.T) { + assert := assert.New(t) + + labelerA := syntax.DID("did:web:aaa.example.com") + labelerB := syntax.DID("did:web:bbb.example.com") + + assert.Equal("", encodeLabelerHeader(nil, nil)) + assert.Equal("did:web:aaa.example.com,did:web:bbb.example.com", encodeLabelerHeader(nil, []syntax.DID{labelerA, labelerB})) + assert.Equal("did:web:aaa.example.com;redact,did:web:bbb.example.com", encodeLabelerHeader([]syntax.DID{labelerA}, []syntax.DID{labelerB})) + assert.Equal("did:web:aaa.example.com;redact", encodeLabelerHeader([]syntax.DID{labelerA}, nil)) +} diff --git a/atproto/atclient/apierror.go b/atproto/atclient/apierror.go new file mode 100644 index 000000000..f00039606 --- /dev/null +++ b/atproto/atclient/apierror.go @@ -0,0 +1,36 @@ +package atclient + +import ( + "fmt" +) + +type APIError struct { + StatusCode int + Name string + Message string +} + +func (ae *APIError) Error() string { + if ae.StatusCode > 0 { + if ae.Name != "" && ae.Message != "" { + return fmt.Sprintf("API request failed (HTTP %d): %s: %s", ae.StatusCode, ae.Name, ae.Message) + } else if ae.Name != "" { + return fmt.Sprintf("API request failed (HTTP %d): %s", ae.StatusCode, ae.Name) + } + return fmt.Sprintf("API request failed (HTTP %d)", ae.StatusCode) + } + return "API request failed" +} + +type ErrorBody struct { + Name string `json:"error"` + Message string `json:"message,omitempty"` +} + +func (eb *ErrorBody) APIError(statusCode int) error { + return &APIError{ + StatusCode: statusCode, + Name: eb.Name, + Message: eb.Message, + } +} diff --git a/atproto/atclient/apirequest.go b/atproto/atclient/apirequest.go new file mode 100644 index 000000000..b676ad830 --- /dev/null +++ b/atproto/atclient/apirequest.go @@ -0,0 +1,117 @@ +package atclient + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +var ( + // atproto API "Query" Lexicon method, which is HTTP GET. Not to be confused with the IETF draft "HTTP QUERY" method. + MethodQuery = http.MethodGet + + // atproto API "Procedure" Lexicon method, which is HTTP POST. + MethodProcedure = http.MethodPost +) + +type APIRequest struct { + // HTTP method as a string (eg "GET") (required) + Method string + + // atproto API endpoint, as NSID (required) + Endpoint syntax.NSID + + // Optional request body (may be nil). If this is provided, then 'Content-Type' header should be specified + Body io.Reader + + // Optional function to return new reader for request body; used for retries. Strongly recommended if Body is defined. Body still needs to be defined, even if this function is provided. + GetBody func() (io.ReadCloser, error) + + // Optional query parameters (field may be nil). These will be encoded as provided. + QueryParams url.Values + + // Optional HTTP headers (field may be nil). Only the first value will be included for each header key ("Set" behavior). + Headers http.Header +} + +// Initializes a new request struct. Initializes Headers and QueryParams so they can be manipulated immediately. +// +// If body is provided (it can be nil), will try to turn it in to the most retry-able form (and wrap as [io.ReadCloser]). +func NewAPIRequest(method string, endpoint syntax.NSID, body io.Reader) *APIRequest { + req := APIRequest{ + Method: method, + Endpoint: endpoint, + Headers: map[string][]string{}, + QueryParams: map[string][]string{}, + } + + // logic to turn "whatever io.Reader we are handed" in to something relatively re-tryable (using GetBody) + if body != nil { + // NOTE: http.NewRequestWithContext already handles GetBody() as well as ContentLength for specific types like bytes.Buffer and strings.Reader. We just want to add io.Seeker here, for things like files-on-disk. + switch v := body.(type) { + case io.Seeker: + req.Body = io.NopCloser(body) + req.GetBody = func() (io.ReadCloser, error) { + v.Seek(0, 0) + return io.NopCloser(body), nil + } + default: + req.Body = body + } + } + return &req +} + +// Creates an [http.Request] for this API request. +// +// `host` parameter should be a URL prefix: schema, hostname, port (required) +// +// `clientHeaders`, if provided, is treated as client-level defaults. Only a single value is allowed per key ("Set" behavior), and will be clobbered by any request-level header values. (optional; may be nil) +func (r *APIRequest) HTTPRequest(ctx context.Context, host string, clientHeaders http.Header) (*http.Request, error) { + u, err := url.Parse(host) + if err != nil { + return nil, err + } + if u.Host == "" { + return nil, fmt.Errorf("empty hostname in host URL") + } + if u.Scheme == "" { + return nil, fmt.Errorf("empty scheme in host URL") + } + if r.Endpoint == "" { + return nil, fmt.Errorf("empty request endpoint") + } + u.Path = "/xrpc/" + r.Endpoint.String() + u.RawQuery = "" + if r.QueryParams != nil && len(r.QueryParams) > 0 { + u.RawQuery = r.QueryParams.Encode() + } + httpReq, err := http.NewRequestWithContext(ctx, r.Method, u.String(), r.Body) + if err != nil { + return nil, err + } + + if r.GetBody != nil { + httpReq.GetBody = r.GetBody + } + + // first set default headers... + if clientHeaders != nil { + for k := range clientHeaders { + httpReq.Header.Set(k, clientHeaders.Get(k)) + } + } + + // ... then request-specific take priority (overwrite) + if r.Headers != nil { + for k := range r.Headers { + httpReq.Header.Set(k, r.Headers.Get(k)) + } + } + + return httpReq, nil +} diff --git a/atproto/atclient/cmd/atp-client-demo/main.go b/atproto/atclient/cmd/atp-client-demo/main.go new file mode 100644 index 000000000..b97d4d1de --- /dev/null +++ b/atproto/atclient/cmd/atp-client-demo/main.go @@ -0,0 +1,244 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + + comatproto "github.com/bluesky-social/indigo/api/agnostic" + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "atp-client-demo", + Usage: "dev helper for atproto/client SDK", + Commands: []*cli.Command{ + &cli.Command{ + Name: "get-feed-public", + Usage: "do a basic GET request (getAuthorFeed)", + Action: runGetFeedPublic, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Value: "https://public.api.bsky.app", + Usage: "service host", + }, + }, + }, + &cli.Command{ + Name: "list-records-public", + Usage: "do a basic GET request (listRecords)", + Action: runListRecordsPublic, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Value: "https://enoki.us-east.host.bsky.network", + Usage: "service host", + }, + }, + }, + &cli.Command{ + Name: "login-auth", + Usage: "do a basic login and GET session info", + Action: runLoginAuth, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "username", + Required: true, + Aliases: []string{"u"}, + Usage: "handle or DID (not email)", + }, + &cli.StringFlag{ + Name: "password", + Required: true, + Aliases: []string{"p"}, + Usage: "password (or app password)", + }, + }, + }, + &cli.Command{ + Name: "get-feed-auth", + Usage: "basic authenticated GET request", + Action: runGetFeedAuth, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "username", + Required: true, + Aliases: []string{"u"}, + Usage: "handle or DID (not email)", + }, + &cli.StringFlag{ + Name: "password", + Required: true, + Aliases: []string{"p"}, + Usage: "password (or app password)", + }, + &cli.StringFlag{ + Name: "labelers", + }, + &cli.StringFlag{ + Name: "appview", + Value: "did:web:api.bsky.app#bsky_appview", + Usage: "bsky appview service DID ref", + }, + }, + }, + &cli.Command{ + Name: "lookup-admin", + Usage: "basic PDS admin auth request (getAccountInfo)", + Action: runLookupAdmin, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "admin-password", + Required: true, + Aliases: []string{"p"}, + Usage: "admin auth password", + }, + &cli.StringFlag{ + Name: "host", + Required: true, + Usage: "service host", + }, + &cli.StringFlag{ + Name: "did", + Required: true, + Usage: "account DID to lookup", + }, + }, + }, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + if err := app.Run(context.Background(), os.Args); err != nil { + slog.Error("command failed", "error", err) + os.Exit(-1) + } +} + +func getFeed(ctx context.Context, c *atclient.APIClient) error { + params := map[string]any{ + "actor": "atproto.com", + "limit": 2, + "includePins": false, + } + + var d json.RawMessage + err := c.Get(ctx, "app.bsky.feed.getAuthorFeed", params, &d) + if err != nil { + return err + } + + out, err := json.MarshalIndent(d, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} + +func listRecords(ctx context.Context, c *atclient.APIClient) error { + + list, err := comatproto.RepoListRecords(ctx, c, "app.bsky.actor.profile", "", 10, "did:plc:ewvi7nxzyoun6zhxrhs64oiz", false) + if err != nil { + return err + } + + out, err := json.MarshalIndent(list, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} + +func runGetFeedPublic(ctx context.Context, cmd *cli.Command) error { + + c := atclient.APIClient{ + Host: cmd.String("host"), + } + + return getFeed(ctx, &c) +} + +func runListRecordsPublic(ctx context.Context, cmd *cli.Command) error { + + c := atclient.APIClient{ + Host: cmd.String("host"), + } + + return listRecords(ctx, &c) +} + +func runLoginAuth(ctx context.Context, cmd *cli.Command) error { + + atid, err := syntax.ParseAtIdentifier(cmd.String("username")) + if err != nil { + return err + } + + dir := identity.DefaultDirectory() + + c, err := atclient.LoginWithPassword(ctx, dir, atid, cmd.String("password"), "", nil) + if err != nil { + return err + } + + var d json.RawMessage + err = c.Get(ctx, "com.atproto.server.getSession", nil, &d) + if err != nil { + return err + } + + out, err := json.MarshalIndent(d, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} + +func runGetFeedAuth(ctx context.Context, cmd *cli.Command) error { + + atid, err := syntax.ParseAtIdentifier(cmd.String("username")) + if err != nil { + return err + } + + dir := identity.DefaultDirectory() + + c, err := atclient.LoginWithPassword(ctx, dir, atid, cmd.String("password"), "", nil) + if err != nil { + return err + } + c = c.WithService(cmd.String("appview")) + + return getFeed(ctx, c) +} + +func runLookupAdmin(ctx context.Context, cmd *cli.Command) error { + + c := atclient.NewAdminClient(cmd.String("host"), cmd.String("admin-password")) + + var d json.RawMessage + params := map[string]any{ + "did": cmd.String("did"), + } + if err := c.Get(ctx, "com.atproto.admin.getAccountInfo", params, &d); err != nil { + return err + } + + out, err := json.MarshalIndent(d, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} diff --git a/atproto/atclient/doc.go b/atproto/atclient/doc.go new file mode 100644 index 000000000..47a5643b6 --- /dev/null +++ b/atproto/atclient/doc.go @@ -0,0 +1,21 @@ +/* +General-purpose client for atproto "XRPC" HTTP API endpoints. + +[APIClient] wraps an [http.Client] and provides an ergonomic atproto-specific (but not Lexicon-specific) interface for "Query" (GET) and "Procedure" (POST) endpoints. It does not support "Event Stream" (WebSocket) endpoints. The client is expected to be used with a single host at a time, though it does have special support ([APIClient.WithService]) for proxied service requests when connected to a PDS host. The client does not authenticate requests by default, but supports pluggable authentication methods (see below). The [APIReponse] struct represents a generic API request, and helps with conversion to an [http.Request]. + +The [APIError] struct can represent a generic API error response (eg, an HTTP response with a 4xx or 5xx response code), including the 'error' and 'message' JSON response fields expected with atproto. It is intended to be used with [errors.Is] in error handling, or to provide helpful error messages. + +The [AuthMethod] interface allows [APIClient] to work with multiple forms of authentication in atproto. It is expected that more complex auth systems (eg, those using signed JWTs) will be implemented in separate packages, but this package does include two simple auth methods: + +- [PasswordAuth] is the original PDS user auth method, using access and refresh tokens. +- [AdminAuth] is simple HTTP Basic authentication for administrative requests, as implemented by many atproto services (Relay, Ozone, PDS, etc). + +## Design Notes + +Several [AuthMethod] implementations are expected to require retrying entire request at unexpected times. For example, unexpected OAuth DPoP nonce changes, or unexpected password session token refreshes. The auth method may also need to make requests to other servers as part of the refresh process (eg, OAuth when working with a PDS/entryway split). This means that requests should be "retryable" as often as possible. This is mostly a concern for Procedures (HTTP POST) with a non-empty body. The [http.Client] will attempt to "unclose" some common [io.ReadCloser] types (like [bytes.Buffer]), but others may need special handling, using the [APIRequest.GetBody] method. This package will try to make types implementing [io.Seeker] tryable; this helps with things like passing in a open file descriptor for file uploads. + +In theory, the [http.RoundTripper] interface could have been used instead of [AuthMethod]; or auth methods could have been injected in to [http.Client] instances directly. This package avoids this pattern for a few reasons. The first is that wrangling layered stacks of [http.RoundTripper] can become cumbersome. Calling code may want to use [http.Client] variants which add observability, retries, circuit-breaking, or other non-auth customization. Secondly, some atproto auth methods will require requests to other servers or endpoints, and having a common [http.Client] to re-use for these requests makes sense. Finally, several atproto auth methods need to know the target endpoint as an NSID; while this could be re-parsed from the request URL, it is simpler and more reliable to pass it as an argument. + +This package tries to use minimal dependencies beyond the Go standard library, to make it easy to reference as a dependency. It does require the [github.com/bluesky-social/indigo/atproto/syntax] and [github.com/bluesky-social/indigo/atproto/identity] sibling packages. In particular, this package does not include any auth methods requiring JWTs, to avoid adding any specific JWT implementation as a dependency. +*/ +package atclient diff --git a/atproto/atclient/examples_test.go b/atproto/atclient/examples_test.go new file mode 100644 index 000000000..4d623dcfa --- /dev/null +++ b/atproto/atclient/examples_test.go @@ -0,0 +1,30 @@ +package atclient + +import ( + "context" + "fmt" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func ExampleGet() { + + ctx := context.Background() + + c := APIClient{ + Host: "https://public.api.bsky.app", + } + + endpoint := syntax.NSID("app.bsky.actor.getProfile") + params := map[string]any{ + "actor": "atproto.com", + } + var profile appbsky.ActorDefs_ProfileViewDetailed + if err := c.Get(ctx, endpoint, params, &profile); err != nil { + panic(err) + } + + fmt.Println(profile.Handle) + // Output: atproto.com +} diff --git a/atproto/atclient/lexclient.go b/atproto/atclient/lexclient.go new file mode 100644 index 000000000..c72d814a0 --- /dev/null +++ b/atproto/atclient/lexclient.go @@ -0,0 +1,92 @@ +package atclient + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Implements the [github.com/bluesky-social/indigo/lex/util.LexClient] interface, for use with code-generated API helpers. +func (c *APIClient) LexDo(ctx context.Context, method string, inputEncoding string, endpoint string, params map[string]any, bodyData any, out any) error { + // some of the code here is copied from indigo:xrpc/xrpc.go + + nsid, err := syntax.ParseNSID(endpoint) + if err != nil { + return err + } + + var body io.Reader + if bodyData != nil { + if rr, ok := bodyData.(io.Reader); ok { + body = rr + } else { + b, err := json.Marshal(bodyData) + if err != nil { + return err + } + + body = bytes.NewReader(b) + if inputEncoding == "" { + inputEncoding = "application/json" + } + } + } + + req := NewAPIRequest(method, nsid, body) + + if inputEncoding != "" { + req.Headers.Set("Content-Type", inputEncoding) + } + + if params != nil { + qp, err := ParseParams(params) + if err != nil { + return err + } + req.QueryParams = qp + } + + resp, err := c.Do(ctx, req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + var eb ErrorBody + if err := json.NewDecoder(resp.Body).Decode(&eb); err != nil { + return &APIError{StatusCode: resp.StatusCode} + } + return eb.APIError(resp.StatusCode) + } + + if out == nil { + // drain body before returning + io.ReadAll(resp.Body) + return nil + } + + if buf, ok := out.(*bytes.Buffer); ok { + if resp.ContentLength < 0 { + _, err := io.Copy(buf, resp.Body) + if err != nil { + return fmt.Errorf("reading response body: %w", err) + } + } else { + n, err := io.CopyN(buf, resp.Body, resp.ContentLength) + if err != nil { + return fmt.Errorf("reading length delimited response body (%d < %d): %w", n, resp.ContentLength, err) + } + } + } else { + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("failed decoding JSON response body: %w", err) + } + } + + return nil +} diff --git a/atproto/atclient/params.go b/atproto/atclient/params.go new file mode 100644 index 000000000..cefba5223 --- /dev/null +++ b/atproto/atclient/params.go @@ -0,0 +1,42 @@ +package atclient + +import ( + "encoding" + "fmt" + "net/url" + "reflect" +) + +// Flexibly parses an input map to URL query params (strings) +func ParseParams(raw map[string]any) (url.Values, error) { + out := make(url.Values) + for k := range raw { + switch v := raw[k].(type) { + case nil: + out.Set(k, "") + case bool, string, int, uint, int8, int16, int32, int64, uint8, uint16, uint32, uint64, uintptr: + out.Set(k, fmt.Sprint(v)) + case encoding.TextMarshaler: + out.Set(k, fmt.Sprint(v)) + default: + ref := reflect.ValueOf(v) + if ref.Kind() == reflect.Slice { + for i := 0; i < ref.Len(); i++ { + switch elem := ref.Index(i).Interface().(type) { + case nil: + out.Add(k, "") + case bool, string, int, uint, int8, int16, int32, int64, uint8, uint16, uint32, uint64, uintptr: + out.Add(k, fmt.Sprint(elem)) + case encoding.TextMarshaler: + out.Add(k, fmt.Sprint(elem)) + default: + return nil, fmt.Errorf("can't marshal query param '%s' with type: %T", k, v) + } + } + } else { + return nil, fmt.Errorf("can't marshal query param '%s' with type: %T", k, v) + } + } + } + return out, nil +} diff --git a/atproto/atclient/params_test.go b/atproto/atclient/params_test.go new file mode 100644 index 000000000..5db429b5e --- /dev/null +++ b/atproto/atclient/params_test.go @@ -0,0 +1,50 @@ +package atclient + +import ( + "net/url" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseParams(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + { + input := map[string]any{ + "int": int(-1), + "uint32": uint32(32), + "str": "hello", + "bool": true, + "did": syntax.DID("did:web:example.com"), + "multiBool": []bool{true, false}, + "multiDID": []syntax.DID{syntax.DID("did:web:example.com"), syntax.DID("did:web:other.com")}, + } + expect := url.Values(map[string][]string{ + "int": []string{"-1"}, + "uint32": []string{"32"}, + "str": []string{"hello"}, + "bool": []string{"true"}, + "did": []string{"did:web:example.com"}, + "multiBool": []string{"true", "false"}, + "multiDID": []string{"did:web:example.com", "did:web:other.com"}, + }) + output, err := ParseParams(input) + require.NoError(err) + assert.Equal(expect, output) + } + + { + // unsupported type + input := map[string]any{ + "map": map[string]int{"a": 123}, + } + _, err := ParseParams(input) + assert.Error(err) + } + +} diff --git a/atproto/atclient/password_auth.go b/atproto/atclient/password_auth.go new file mode 100644 index 000000000..a2623a167 --- /dev/null +++ b/atproto/atclient/password_auth.go @@ -0,0 +1,304 @@ +package atclient + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "sync" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type RefreshCallback = func(ctx context.Context, data PasswordSessionData) + +// Implementation of [AuthMethod] for password-based auth sessions with atproto PDS hosts. Automatically refreshes "access token" using a "refresh token" when needed. +// +// It is safe to use this auth method concurrently from multiple goroutines. +type PasswordAuth struct { + Session PasswordSessionData + + // Optional callback function which gets called with updated session data whenever a successful token refresh happens. + // + // Note that this function is called while a lock is being held on the overall client, and with a context usually tied to a regular API request call. The callback should either return quickly, or spawn a goroutine. Because of the lock, this callback will never be called concurrently for a single client, but may be called currently across clients. + RefreshCallback RefreshCallback + + // Lock which protects concurrent access to AccessToken and RefreshToken in session data. Note that this only applies to this particular instance of PasswordAuth. + lk sync.RWMutex +} + +// Data about a PDS password auth session which can be persisted and then used to resume the session later. +type PasswordSessionData struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + AccountDID syntax.DID `json:"account_did"` + Host string `json:"host"` +} + +// Creates a deep copy of the session data. +func (sd *PasswordSessionData) Clone() PasswordSessionData { + return PasswordSessionData{ + AccessToken: sd.AccessToken, + RefreshToken: sd.RefreshToken, + AccountDID: sd.AccountDID, + Host: sd.Host, + } +} + +type createSessionRequest struct { + //AllowTakendown *bool `json:"allowTakendown,omitempty" cborgen:"allowTakendown,omitempty"` + AuthFactorToken *string `json:"authFactorToken,omitempty" cborgen:"authFactorToken,omitempty"` + // identifier: Handle or other identifier supported by the server for the authenticating user. + Identifier string `json:"identifier" cborgen:"identifier"` + Password string `json:"password" cborgen:"password"` +} + +type createSessionResponse struct { + AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` + Active *bool `json:"active,omitempty" cborgen:"active,omitempty"` + Did string `json:"did" cborgen:"did"` + //Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + //EmailAuthFactor *bool `json:"emailAuthFactor,omitempty" cborgen:"emailAuthFactor,omitempty"` + //EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"` + //Handle string `json:"handle" cborgen:"handle"` + RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +type refreshSessionResponse struct { + AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` + Active *bool `json:"active,omitempty" cborgen:"active,omitempty"` + Did string `json:"did" cborgen:"did"` + //Handle string `json:"handle" cborgen:"handle"` + RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` + Status *string `json:"status,omitempty" cborgen:"status,omitempty"` +} + +func (a *PasswordAuth) DoWithAuth(c *http.Client, req *http.Request, endpoint syntax.NSID) (*http.Response, error) { + accessToken, refreshToken := a.GetTokens() + req.Header.Set("Authorization", "Bearer "+accessToken) + resp, err := c.Do(req) + if err != nil { + return nil, err + } + + // on success, or most errors, just return HTTP response + if resp.StatusCode != http.StatusBadRequest || !strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") { + return resp, nil + } + + // parse the error response body (JSON) and check the error name + defer resp.Body.Close() + var eb ErrorBody + if err := json.NewDecoder(resp.Body).Decode(&eb); err != nil { + return nil, &APIError{StatusCode: resp.StatusCode} + } + if eb.Name != "ExpiredToken" { + return nil, eb.APIError(resp.StatusCode) + } + + // ok, we had an expired token, try a refresh + if err := a.Refresh(req.Context(), c, refreshToken); err != nil { + return nil, err + } + + retry := req.Clone(req.Context()) + if req.GetBody != nil { + retry.Body, err = req.GetBody() + if err != nil { + return nil, fmt.Errorf("API request retry GetBody failed: %w", err) + } + } + + accessToken, _ = a.GetTokens() + + retry.Header.Set("Authorization", "Bearer "+accessToken) + retryResp, err := c.Do(retry) + if err != nil { + return nil, err + } + // NOTE: could handle auth failure as special error type here + return retryResp, err +} + +// Returns current access and refresh tokens (take a read-lock on session data) +func (a *PasswordAuth) GetTokens() (string, string) { + a.lk.RLock() + defer a.lk.RUnlock() + return a.Session.AccessToken, a.Session.RefreshToken +} + +// Refreshes auth tokens (takes a write-lock on session data). +// +// `priorRefreshToken` argument is used to check if a concurrent refresh already took place. +func (a *PasswordAuth) Refresh(ctx context.Context, c *http.Client, priorRefreshToken string) error { + + a.lk.Lock() + defer a.lk.Unlock() + + // basic concurrency check: if refresh token already changed, can bail here (releasing lock) + if priorRefreshToken != "" && priorRefreshToken != a.Session.RefreshToken { + return nil + } + + u := a.Session.Host + "/xrpc/com.atproto.server.refreshSession" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, nil) + if err != nil { + return err + } + // NOTE: could try to pull User-Agent from a request and pass that through to here + req.Header.Set("User-Agent", "indigo-sdk") + + // NOTE: using refresh token here, not access token + req.Header.Set("Authorization", "Bearer "+a.Session.RefreshToken) + + resp, err := c.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + var eb ErrorBody + if err := json.NewDecoder(resp.Body).Decode(&eb); err != nil { + return &APIError{StatusCode: resp.StatusCode} + } + // TODO: indicate in the error that it was from refresh process, not original request? + return eb.APIError(resp.StatusCode) + } + + var out refreshSessionResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + a.Session.AccessToken = out.AccessJwt + a.Session.RefreshToken = out.RefreshJwt + + if a.RefreshCallback != nil { + snapshot := a.Session.Clone() + a.RefreshCallback(ctx, snapshot) + } + + return nil +} + +func (a *PasswordAuth) Logout(ctx context.Context, c *http.Client) error { + _, refreshToken := a.GetTokens() + + u := a.Session.Host + "/xrpc/com.atproto.server.deleteSession" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, nil) + if err != nil { + return err + } + // NOTE: could try to pull User-Agent from a request and pass that through to here + req.Header.Set("User-Agent", "indigo-sdk") + + // NOTE: using refresh token here, not access token + req.Header.Set("Authorization", "Bearer "+refreshToken) + + resp, err := c.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + var eb ErrorBody + if err := json.NewDecoder(resp.Body).Decode(&eb); err != nil { + return &APIError{StatusCode: resp.StatusCode} + } + return eb.APIError(resp.StatusCode) + } + return nil +} + +// Creates a new [APIClient] with [PasswordAuth] for the provided user. The provided identity directory is used to resolve the PDS host for the account. +// +// `authToken` is optional; is used when multi-factor authentication is enabled for the account. +// +// `cb` is an optional callback which will be called with updated session data after any token refresh. +func LoginWithPassword(ctx context.Context, dir identity.Directory, username syntax.AtIdentifier, password, authToken string, cb RefreshCallback) (*APIClient, error) { + + ident, err := dir.Lookup(ctx, username) + if err != nil { + return nil, err + } + + host := ident.PDSEndpoint() + if host == "" { + return nil, fmt.Errorf("account does not have PDS registered") + } + + c, err := LoginWithPasswordHost(ctx, host, ident.DID.String(), password, authToken, cb) + if err != nil { + return nil, err + } + + if c.AccountDID == nil || *c.AccountDID != ident.DID { + return nil, fmt.Errorf("returned session DID not requested account: %s", c.AccountDID) + } + + return c, nil +} + +// Creates a new [APIClient] with [PasswordAuth], based on a login to the provided host. Note that with some PDS implementations, 'username' could be an email address. This login method also works in situations where an account's network identity does not resolve to this specific host. +// +// `authToken` is optional; is used when multi-factor authentication is enabled for the account. +// +// `cb` is an optional callback which will be called with updated session data after any token refresh. +func LoginWithPasswordHost(ctx context.Context, host, username, password, authToken string, cb RefreshCallback) (*APIClient, error) { + + c := NewAPIClient(host) + reqBody := createSessionRequest{ + Identifier: username, + Password: password, + } + if authToken != "" { + reqBody.AuthFactorToken = &authToken + } + + var out createSessionResponse + if err := c.Post(ctx, syntax.NSID("com.atproto.server.createSession"), &reqBody, &out); err != nil { + return nil, err + } + + if out.Active != nil && *out.Active == false { + slog.Info("password login to inactive account", "status", *out.Status, "username", username) + } + + did, err := syntax.ParseDID(out.Did) + if err != nil { + return nil, err + } + + ra := PasswordAuth{ + Session: PasswordSessionData{ + AccessToken: out.AccessJwt, + RefreshToken: out.RefreshJwt, + AccountDID: did, + Host: c.Host, + }, + RefreshCallback: cb, + } + c.Auth = &ra + c.AccountDID = &did + return c, nil +} + +// Creates an [APIClient] using [PasswordAuth], based on existing session data. +// +// `cb` is an optional callback which will be called with updated session data after any token refresh. +func ResumePasswordSession(data PasswordSessionData, cb RefreshCallback) *APIClient { + c := NewAPIClient(data.Host) + ra := PasswordAuth{ + Session: data, + RefreshCallback: cb, + } + c.Auth = &ra + c.AccountDID = &data.AccountDID + return c +} diff --git a/atproto/atclient/password_auth_test.go b/atproto/atclient/password_auth_test.go new file mode 100644 index 000000000..6f8012075 --- /dev/null +++ b/atproto/atclient/password_auth_test.go @@ -0,0 +1,244 @@ +package atclient + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func pwHandler(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/xrpc/com.atproto.server.refreshSession": + //fmt.Println("refreshSession handler...") + hdr := r.Header.Get("Authorization") + if hdr != "Bearer refresh1" { + fmt.Printf("refreshSession header: %s\n", hdr) + w.Header().Set("WWW-Authenticate", `Bearer`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]string{ + "did": "did:web:account.example.com", + "accessJwt": "access2", + "refreshJwt": "refresh2", + }) + return + case "/xrpc/com.atproto.server.deleteSession": + //fmt.Println("deleteSession handler...") + hdr := r.Header.Get("Authorization") + if hdr != "Bearer refresh1" { + fmt.Printf("refreshSession header: %s\n", hdr) + w.Header().Set("WWW-Authenticate", `Bearer`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + return + case "/xrpc/com.atproto.server.createSession": + if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + fmt.Println("createSession Content-Type") + http.Error(w, "Bad Request", http.StatusBadRequest) + return + } + var body map[string]string + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + fmt.Println("createSession JSON") + http.Error(w, "Bad Request", http.StatusBadRequest) + return + } + if body["identifier"] != "did:web:account.example.com" || body["password"] != "password1" { + fmt.Println("createSession wrong password") + http.Error(w, "Bad Request", http.StatusUnauthorized) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]string{ + "did": body["identifier"], + "accessJwt": "access1", + "refreshJwt": "refresh1", + }) + return + case "/xrpc/com.example.get", "/xrpc/com.example.post": + hdr := r.Header.Get("Authorization") + if hdr == "Bearer access1" || hdr == "Bearer access2" { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintln(w, "{\"status\":\"success\"}") + return + } else { + fmt.Printf("get header: %s\n", hdr) + w.Header().Set("WWW-Authenticate", `Bearer`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + case "/xrpc/com.example.expire": + hdr := r.Header.Get("Authorization") + if hdr == "Bearer access1" { + //fmt.Println("forcing refresh...") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + fmt.Fprintln(w, "{\"error\":\"ExpiredToken\"}") + return + } else if hdr == "Bearer access2" { + w.Header().Set("Content-Type", "application/json") + fmt.Fprintln(w, "{\"status\":\"success\"}") + return + } else { + fmt.Printf("expire header: %s\n", hdr) + w.Header().Set("WWW-Authenticate", `Bearer`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + default: + http.NotFound(w, r) + return + } +} + +func TestPasswordAuth(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + ctx := context.Background() + + srv := httptest.NewServer(http.HandlerFunc(pwHandler)) + defer srv.Close() + + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{ + DID: "did:web:account.example.com", + Handle: "user1.example.com", + Services: map[string]identity.ServiceEndpoint{ + "atproto_pds": { + Type: "AtprotoPersonalDataServer", + URL: srv.URL, + }, + }, + }) + + { + // simple GET requests, with token expire/retry + c, err := LoginWithPassword(ctx, &dir, syntax.Handle("user1.example.com").AtIdentifier(), "password1", "", nil) + require.NoError(err) + err = c.Get(ctx, syntax.NSID("com.example.get"), nil, nil) + assert.NoError(err) + err = c.Get(ctx, syntax.NSID("com.example.expire"), nil, nil) + assert.NoError(err) + } + + { + // test resume session, and session data callback mechanism + ch := make(chan string, 10) + cb := func(ctx context.Context, data PasswordSessionData) { + assert.Equal("refresh2", data.RefreshToken) + ch <- "refreshed" + } + c := ResumePasswordSession(PasswordSessionData{ + AccessToken: "access1", + RefreshToken: "refresh1", + AccountDID: syntax.DID("did:web:account.example.com"), + Host: srv.URL, + }, cb) + + err := c.Get(ctx, syntax.NSID("com.example.get"), nil, nil) + assert.NoError(err) + err = c.Get(ctx, syntax.NSID("com.example.expire"), nil, nil) + assert.NoError(err) + + select { + case msg := <-ch: + assert.Equal("refreshed", msg) + } + } + + { + // logout + c, err := LoginWithPassword(ctx, &dir, syntax.Handle("user1.example.com").AtIdentifier(), "password1", "", nil) + require.NoError(err) + + passAuth, ok := c.Auth.(*PasswordAuth) + require.True(ok) + err = passAuth.Logout(ctx, c.Client) + assert.NoError(err) + } + + { + // simple POST request, with token expire/retry + c, err := LoginWithPassword(ctx, &dir, syntax.Handle("user1.example.com").AtIdentifier(), "password1", "", nil) + require.NoError(err) + body := map[string]any{ + "a": 123, + "b": "hello", + } + var out json.RawMessage + err = c.Post(ctx, syntax.NSID("com.example.post"), body, &out) + assert.NoError(err) + err = c.Post(ctx, syntax.NSID("com.example.expire"), body, &out) + assert.NoError(err) + } + + { + // POST with bytes.Buffer body + c, err := LoginWithPassword(ctx, &dir, syntax.Handle("user1.example.com").AtIdentifier(), "password1", "", nil) + require.NoError(err) + body := bytes.NewBufferString("some text") + req := NewAPIRequest(MethodProcedure, syntax.NSID("com.example.expire"), body) + req.Headers.Set("Content-Type", "text/plain") + resp, err := c.Do(ctx, req) + require.NoError(err) + assert.Equal(200, resp.StatusCode) + } + + { + // POST with file on disk (can seek and retry) + c, err := LoginWithPassword(ctx, &dir, syntax.Handle("user1.example.com").AtIdentifier(), "password1", "", nil) + require.NoError(err) + f, err := os.Open("testdata/body.json") + require.NoError(err) + req := NewAPIRequest(MethodProcedure, syntax.NSID("com.example.expire"), f) + req.Headers.Set("Content-Type", "application/json") + resp, err := c.Do(ctx, req) + require.NoError(err) + assert.Equal(200, resp.StatusCode) + } + + { + // POST with pipe reader (can *not* retry) + c, err := LoginWithPassword(ctx, &dir, syntax.Handle("user1.example.com").AtIdentifier(), "password1", "", nil) + require.NoError(err) + r1, w1 := io.Pipe() + go func() { + fmt.Fprintf(w1, "some data") + w1.Close() + }() + req1 := NewAPIRequest(MethodProcedure, syntax.NSID("com.example.post"), r1) + req1.Headers.Set("Content-Type", "text/plain") + resp, err := c.Do(ctx, req1) + require.NoError(err) + assert.Equal(200, resp.StatusCode) + + // expect this to fail (can't re-read from Pipe) + r2, w2 := io.Pipe() + go func() { + fmt.Fprintf(w2, "some data") + w2.Close() + }() + req2 := NewAPIRequest(MethodProcedure, syntax.NSID("com.example.expire"), r2) + req2.Headers.Set("Content-Type", "text/plain") + _, err = c.Do(ctx, req2) + assert.Error(err) + } +} diff --git a/atproto/atclient/testdata/body.json b/atproto/atclient/testdata/body.json new file mode 100644 index 000000000..01d7dc358 --- /dev/null +++ b/atproto/atclient/testdata/body.json @@ -0,0 +1,4 @@ +{ + "a": 123, + "b": "hello" +} diff --git a/atproto/atcrypto/cmd/atp-crypto/main.go b/atproto/atcrypto/cmd/atp-crypto/main.go new file mode 100644 index 000000000..036064adc --- /dev/null +++ b/atproto/atcrypto/cmd/atp-crypto/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "atp-crypto", + Usage: "informal debugging CLI tool for atproto key and cryptography", + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "generate", + Usage: "create a new private key", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "p256", + Usage: "generate a P-256 / secp256r1 / ES256 private key (default)", + }, + &cli.BoolFlag{ + Name: "k256", + Usage: "generate a K-256 / secp256k1 / ES256K private key", + }, + }, + + Action: runGenerate, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + if err := app.Run(context.Background(), os.Args); err != nil { + slog.Error("command failed", "error", err) + os.Exit(-1) + } +} + +func runGenerate(ctx context.Context, cmd *cli.Command) error { + if cmd.Bool("k256") { + priv, err := atcrypto.GeneratePrivateKeyK256() + if err != nil { + return err + } + fmt.Println(priv.Multibase()) + } else { + priv, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + return err + } + fmt.Println(priv.Multibase()) + } + return nil +} diff --git a/atproto/atcrypto/docs.go b/atproto/atcrypto/docs.go new file mode 100644 index 000000000..20f7602a9 --- /dev/null +++ b/atproto/atcrypto/docs.go @@ -0,0 +1,11 @@ +// Package atcrypto provides cryptographic keys and operations, as used in atproto (the protocol) +// +// This package attempts to abstract away the specific curves, compressions, signature variations, and other implementation details. The goal is to provide as few knobs and options as possible when working with this library. Use of cryptography in atproto is specified in https://atproto.com/specs/cryptography. +// +// The two currently supported curve types are: +// +// - P-256/secp256r1, internally implemented using golang's stdlib cryptographic library +// - K-256/secp256r1, internally implemented using https://gitlab.com/yawning/secp256k1-voi +// +// "Low-S" signatures are enforced for both key types, both when creating signatures and during verification, as required by the atproto specification. +package atcrypto diff --git a/atproto/atcrypto/examples_test.go b/atproto/atcrypto/examples_test.go new file mode 100644 index 000000000..088626b78 --- /dev/null +++ b/atproto/atcrypto/examples_test.go @@ -0,0 +1,47 @@ +package atcrypto + +import ( + "encoding/base64" + "fmt" +) + +func ExamplePublicKey() { + pub, err := ParsePublicDIDKey("did:key:zDnaembgSGUhZULN2Caob4HLJPaxBh92N7rtH21TErzqf8HQo") + if err != nil { + panic("failed to parse did:key") + } + + // parse existing base64 message and signature to raw bytes + msg, _ := base64.RawStdEncoding.DecodeString("oWVoZWxsb2V3b3JsZA") + sig, _ := base64.RawStdEncoding.DecodeString("2vZNsG3UKvvO/CDlrdvyZRISOFylinBh0Jupc6KcWoJWExHptCfduPleDbG3rko3YZnn9Lw0IjpixVmexJDegg") + if err = pub.HashAndVerify(msg, sig); err != nil { + fmt.Println("Verification Failed") + } else { + fmt.Println("Success!") + } + // Output: Success! +} + +func ExamplePrivateKey() { + // create secure private key, and corresponding public key + priv, err := GeneratePrivateKeyK256() + if err != nil { + panic("failed to generate key") + } + pub, err := priv.PublicKey() + if err != nil { + panic("failed to get public key") + } + + // sign a message + msg := []byte("hello world") + sig, _ := priv.HashAndSign(msg) + + // verify the message + if err = pub.HashAndVerify(msg, sig); err != nil { + fmt.Println("Verification Failed") + } else { + fmt.Println("Success!") + } + // Output: Success! +} diff --git a/atproto/atcrypto/interop_fixtures_test.go b/atproto/atcrypto/interop_fixtures_test.go new file mode 100644 index 000000000..7c91b1685 --- /dev/null +++ b/atproto/atcrypto/interop_fixtures_test.go @@ -0,0 +1,91 @@ +package atcrypto + +import ( + "encoding/base64" + "encoding/json" + "io" + "os" + "testing" + + "github.com/mr-tron/base58" + "github.com/stretchr/testify/assert" +) + +type InteropFixture struct { + MessageBase64 string `json:"messageBase64"` + Algorithm string `json:"algorithm"` + DIDDocSuite string `json:"didDocSuite"` + PublicKeyDID string `json:"publicKeyDid"` + PublicKeyMultibase string `json:"publicKeyMultibase"` + SignatureBase64 string `json:"signatureBase64"` + ValidSignature bool `json:"validSignature"` +} + +func TestInteropSignatureFixtures(t *testing.T) { + // "p256" == "secp256r1" == "ES256" == "EcdsaSecp256r1VerificationKey2019" + // "k256" == "secp256k1" == "ES256K" == "EcdsaSecp256k1VerificationKey2019" + + f, err := os.Open("testdata/signature-fixtures.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + fixBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []InteropFixture + if err := json.Unmarshal(fixBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, row := range fixtures { + _ = row + testSignatureFixture(t, row) + } +} + +func testSignatureFixture(t *testing.T, row InteropFixture) { + assert := assert.New(t) + + // parse all the fields + pkDID, err := ParsePublicDIDKey(row.PublicKeyDID) + assert.NoError(err) + keyBytes, err := base58.Decode(row.PublicKeyMultibase[1:]) + assert.NoError(err) + msgBytes, err := base64.RawStdEncoding.DecodeString(row.MessageBase64) + assert.NoError(err) + sigBytes, err := base64.RawStdEncoding.DecodeString(row.SignatureBase64) + assert.NoError(err) + + var pkCompMultibase PublicKey + switch row.DIDDocSuite { + case "EcdsaSecp256r1VerificationKey2019": + pkCompMultibase, err = ParsePublicBytesP256(keyBytes) + assert.NoError(err) + case "EcdsaSecp256k1VerificationKey2019": + pkCompMultibase, err = ParsePublicBytesK256(keyBytes) + assert.NoError(err) + default: + t.Fatal("expected DIDDocSuite") + } + + // verify encodings + assert.Equal(pkDID, pkCompMultibase, "key equality") + assert.Equal(row.PublicKeyDID, pkDID.DIDKey(), "did:key re-encoding") + + // verify signatures + if row.ValidSignature { + assert.NoError(pkDID.HashAndVerify(msgBytes, sigBytes), "keyType=%v format=%v", row.Algorithm, "did:key") + assert.NoError(pkCompMultibase.HashAndVerify(msgBytes, sigBytes), "keyType=%v format=%v", row.Algorithm, "multibase") + } else { + assert.Error(pkDID.HashAndVerify(msgBytes, sigBytes), "keyType=%v format=%v", row.Algorithm, "did:key") + assert.Error(pkCompMultibase.HashAndVerify(msgBytes, sigBytes), "keyType=%v format=%v", row.Algorithm, "multibase") + } + + // signatures don't match random data + assert.Error(pkCompMultibase.HashAndVerify(msgBytes, []byte{1, 2, 3}), "keyType=%v format=%v", row.Algorithm, "multibase") + assert.Error(pkCompMultibase.HashAndVerify([]byte{1, 2, 3}, sigBytes), "keyType=%v format=%v", row.Algorithm, "multibase") +} diff --git a/atproto/atcrypto/jwk.go b/atproto/atcrypto/jwk.go new file mode 100644 index 000000000..f51b725a3 --- /dev/null +++ b/atproto/atcrypto/jwk.go @@ -0,0 +1,124 @@ +package atcrypto + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "encoding/base64" + "encoding/json" + "fmt" + "math/big" + + secp256k1 "gitlab.com/yawning/secp256k1-voi" + secp256k1secec "gitlab.com/yawning/secp256k1-voi/secec" +) + +// Representation of a JSON Web Key (JWK), as relevant to the keys supported by this package. +// +// Expected to be marshalled/unmarshalled as JSON. +type JWK struct { + KeyType string `json:"kty"` + Curve string `json:"crv"` + X string `json:"x"` // base64url, no padding + Y string `json:"y"` // base64url, no padding + Use string `json:"use,omitempty"` + KeyID *string `json:"kid,omitempty"` +} + +// Loads a [PublicKey] from JWK (serialized as JSON bytes) +func ParsePublicJWKBytes(jwkBytes []byte) (PublicKey, error) { + var jwk JWK + if err := json.Unmarshal(jwkBytes, &jwk); err != nil { + return nil, fmt.Errorf("parsing JWK JSON: %w", err) + } + return ParsePublicJWK(jwk) +} + +// Loads a [PublicKey] from JWK struct. +func ParsePublicJWK(jwk JWK) (PublicKey, error) { + + if jwk.KeyType != "EC" { + return nil, fmt.Errorf("unsupported JWK key type: %s", jwk.KeyType) + } + + // base64url with no encoding + xbuf, err := base64.RawURLEncoding.DecodeString(jwk.X) + if err != nil { + return nil, fmt.Errorf("invalid JWK base64 encoding: %w", err) + } + ybuf, err := base64.RawURLEncoding.DecodeString(jwk.Y) + if err != nil { + return nil, fmt.Errorf("invalid JWK base64 encoding: %w", err) + } + + switch jwk.Curve { + case "P-256": + curve := elliptic.P256() + + var x, y big.Int + x.SetBytes(xbuf) + y.SetBytes(ybuf) + + if !curve.Params().IsOnCurve(&x, &y) { + return nil, fmt.Errorf("invalid P-256 public key (not on curve)") + } + pubECDSA := &ecdsa.PublicKey{ + Curve: curve, + X: &x, + Y: &y, + } + pub := PublicKeyP256{pubP256: *pubECDSA} + err := pub.checkCurve() + if err != nil { + return nil, err + } + return &pub, nil + case "secp256k1": // K-256 + if len(xbuf) != 32 || len(ybuf) != 32 { + return nil, fmt.Errorf("invalid K-256 coordinates") + } + xarr := ([32]byte)(xbuf[:32]) + yarr := ([32]byte)(ybuf[:32]) + p, err := secp256k1.NewPointFromCoords(&xarr, &yarr) + if err != nil { + return nil, fmt.Errorf("invalid K-256 coordinates: %w", err) + } + pubK, err := secp256k1secec.NewPublicKeyFromPoint(p) + if err != nil { + return nil, fmt.Errorf("invalid K-256/secp256k1 public key: %w", err) + } + pub := PublicKeyK256{pubK256: pubK} + err = pub.ensureBytes() + if err != nil { + return nil, err + } + return &pub, nil + default: + return nil, fmt.Errorf("unsupported JWK cryptography: %s", jwk.Curve) + } +} + +func (k *PublicKeyP256) JWK() (*JWK, error) { + jwk := JWK{ + KeyType: "EC", + Curve: "P-256", + X: base64.RawURLEncoding.EncodeToString(k.pubP256.X.Bytes()), + Y: base64.RawURLEncoding.EncodeToString(k.pubP256.Y.Bytes()), + } + return &jwk, nil +} + +func (k *PublicKeyK256) JWK() (*JWK, error) { + raw := k.UncompressedBytes() + if len(raw) != 65 { + return nil, fmt.Errorf("unexpected K-256 bytes size") + } + xbytes := raw[1:33] + ybytes := raw[33:65] + jwk := JWK{ + KeyType: "EC", + Curve: "secp256k1", + X: base64.RawURLEncoding.EncodeToString(xbytes), + Y: base64.RawURLEncoding.EncodeToString(ybytes), + } + return &jwk, nil +} diff --git a/atproto/atcrypto/jwk_test.go b/atproto/atcrypto/jwk_test.go new file mode 100644 index 000000000..12c4957fb --- /dev/null +++ b/atproto/atcrypto/jwk_test.go @@ -0,0 +1,98 @@ +package atcrypto + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseJWK(t *testing.T) { + assert := assert.New(t) + + jwkTestFixtures := []string{ + // https://datatracker.ietf.org/doc/html/rfc7517#appendix-A.1 + `{ + "kty":"EC", + "crv":"P-256", + "x":"MKBCTNIcKUSDii11ySs3526iDZ8AiTo7Tu6KPAqv7D4", + "y":"4Etl6SRW2YiLUrN5vfvVHuhp7x8PxltmWWlbbM4IFyM", + "d":"870MB6gfuTJ4HtUnUvYMyJpr5eUZNP4Bk43bVdj3eAE", + "use":"enc", + "kid":"1" + }`, + // https://openid.net/specs/draft-jones-json-web-key-03.html; with kty in addition to alg + `{ + "alg":"EC", + "kty":"EC", + "crv":"P-256", + "x":"MKBCTNIcKUSDii11ySs3526iDZ8AiTo7Tu6KPAqv7D4", + "y":"4Etl6SRW2YiLUrN5vfvVHuhp7x8PxltmWWlbbM4IFyM", + "use":"enc", + "kid":"1" + }`, + // https://w3c-ccg.github.io/lds-ecdsa-secp256k1-2019/; with kty in addition to alg + `{ + "alg": "EC", + "kty": "EC", + "crv": "secp256k1", + "kid": "JUvpllMEYUZ2joO59UNui_XYDqxVqiFLLAJ8klWuPBw", + "x": "dWCvM4fTdeM0KmloF57zxtBPXTOythHPMm1HCLrdd3A", + "y": "36uMVGM7hnw-N6GnjFcihWE3SkrhMLzzLCdPMXPEXlA" + }`, + } + + for _, jwkBytes := range jwkTestFixtures { + _, err := ParsePublicJWKBytes([]byte(jwkBytes)) + assert.NoError(err) + } +} + +func TestP256GenJWK(t *testing.T) { + assert := assert.New(t) + + priv, err := GeneratePrivateKeyP256() + if err != nil { + t.Fatal(err) + } + pub, err := priv.PublicKey() + if err != nil { + t.Fatal(err) + } + + pk, ok := pub.(*PublicKeyP256) + if !ok { + t.Fatal() + } + jwk, err := pk.JWK() + if err != nil { + t.Fatal(err) + } + + _, err = ParsePublicJWK(*jwk) + assert.NoError(err) +} + +func TestK256GenJWK(t *testing.T) { + assert := assert.New(t) + + priv, err := GeneratePrivateKeyK256() + if err != nil { + t.Fatal(err) + } + pub, err := priv.PublicKey() + if err != nil { + t.Fatal(err) + } + + pk, ok := pub.(*PublicKeyK256) + if !ok { + t.Fatal() + } + jwk, err := pk.JWK() + if err != nil { + t.Fatal(err) + } + + _, err = ParsePublicJWK(*jwk) + assert.NoError(err) +} diff --git a/atproto/atcrypto/k256.go b/atproto/atcrypto/k256.go new file mode 100644 index 000000000..c89bf2f18 --- /dev/null +++ b/atproto/atcrypto/k256.go @@ -0,0 +1,227 @@ +package atcrypto + +import ( + "crypto" + "crypto/rand" + "crypto/sha256" + "fmt" + + "github.com/mr-tron/base58" + secp256k1 "gitlab.com/yawning/secp256k1-voi" + secp256k1secec "gitlab.com/yawning/secp256k1-voi/secec" +) + +// Implements the [PrivateKeyExportable] and [PrivateKey] interfaces for the NIST K-256 / secp256k1 / ES256K cryptographic curve. +// Secret key material is naively stored in memory. +type PrivateKeyK256 struct { + privK256 *secp256k1secec.PrivateKey +} + +// K-256 / secp256k1 / ES256K +// Implements the [PublicKey] interface for the NIST K-256 / secp256k1 / ES256K cryptographic curve. +type PublicKeyK256 struct { + pubK256 *secp256k1secec.PublicKey +} + +var _ PrivateKey = (*PrivateKeyK256)(nil) +var _ PrivateKeyExportable = (*PrivateKeyK256)(nil) +var _ PublicKey = (*PublicKeyK256)(nil) + +var k256Options = &secp256k1secec.ECDSAOptions{ + // Used to *verify* digest, not to re-hash + Hash: crypto.SHA256, + // Use `[R | S]` encoding. + Encoding: secp256k1secec.EncodingCompact, + // Checking `s <= n/2` to prevent signature mallability is not part of SEC 1, Version 2.0. libsecp256k1 which used to be used by this package, includes the check, so retain behavior compatibility. + RejectMalleable: true, +} + +var k256LenientOptions = &secp256k1secec.ECDSAOptions{ + // Used to *verify* digest, not to re-hash + Hash: crypto.SHA256, + // Use `[R | S]` encoding. + Encoding: secp256k1secec.EncodingCompact, + // Allows (eg, for JWT validation) + RejectMalleable: false, +} + +// Creates a secure new cryptographic key from scratch, with the indicated curve type. +func GeneratePrivateKeyK256() (*PrivateKeyK256, error) { + key, err := secp256k1secec.GenerateKey() + if err != nil { + return nil, fmt.Errorf("K-256/secp256k1 key generation failed: %w", err) + } + return &PrivateKeyK256{privK256: key}, nil +} + +// Loads a [PrivateKeyK256] from raw bytes, as exported by the PrivateKey.Bytes method. +// +// Calling code needs to know the key type ahead of time, and must remove any string encoding (hex encoding, base64, etc) before calling this function. +func ParsePrivateBytesK256(data []byte) (*PrivateKeyK256, error) { + sk, err := secp256k1secec.NewPrivateKey(data) + if err != nil { + return nil, fmt.Errorf("invalid K-256/secp256k1 private key: %w", err) + } + return &PrivateKeyK256{privK256: sk}, nil +} + +// Checks if the two private keys are the same. Note that the naive == operator does not work for most equality checks. +func (k *PrivateKeyK256) Equal(other PrivateKey) bool { + otherK256, ok := other.(*PrivateKeyK256) + if ok { + return k.privK256.Equal(otherK256.privK256) + } + return false +} + +// Serializes the secret key material in to a raw binary format, which can be parsed by [ParsePrivateBytesK256]. +// +// For K-256, this is the "compact" encoding and is 32 bytes long. There is no ASN.1 or other enclosing structure. +func (k PrivateKeyK256) Bytes() []byte { + return k.privK256.Bytes() +} + +// Multibase string encoding of the private key, including a multicodec indicator +func (k *PrivateKeyK256) Multibase() string { + kbytes := k.Bytes() + // multicodec secp256k1-priv, code 0x1301, varint-encoded bytes: [0x81, 0x26] + kbytes = append([]byte{0x81, 0x26}, kbytes...) + return "z" + base58.Encode(kbytes) +} + +// Outputs the [PublicKey] corresponding to this [PrivateKeyK256]; it will be a [PublicKeyK256]. +func (k PrivateKeyK256) PublicKey() (PublicKey, error) { + pub := PublicKeyK256{pubK256: k.privK256.PublicKey()} + err := pub.ensureBytes() + if err != nil { + return nil, err + } + return &pub, nil +} + +// First hashes the raw bytes, then signs the digest, returning a binary signature. +// +// SHA-256 is the hash algorithm used, as specified by atproto. Signing digests is the norm for ECDSA, and required by some backend implementations. This method does not "double hash", it simply has name which clarifies that hashing is happening. +// +// Calling code is responsible for any string encoding of signatures (eg, hex or base64). For K-256, the signature is 64 bytes long. +// +// NIST ECDSA signatures can have a "malleability" issue, meaning that there are multiple valid signatures for the same content with the same signing key. This method always returns a "low-S" signature, as required by atproto. +func (k PrivateKeyK256) HashAndSign(content []byte) ([]byte, error) { + hash := sha256.Sum256(content) + return k.privK256.Sign(rand.Reader, hash[:], k256Options) +} + +// Loads a [PublicKeyK256] raw bytes, as exported by the PublicKey.Bytes method. This is the "compressed" curve format. +// +// Calling code needs to know the key type ahead of time, and must remove any string encoding (hex encoding, base64, etc) before calling this function. +func ParsePublicBytesK256(data []byte) (*PublicKeyK256, error) { + // secp256k1secec.NewPublicKey accepts any valid encoding, while we + // explicitly want compressed, so use the explicit point + // decompression routine. + p, err := secp256k1.NewIdentityPoint().SetCompressedBytes(data) + if err != nil { + return nil, fmt.Errorf("invalid K-256/secp256k1 public key: %w", err) + } + + pubK, err := secp256k1secec.NewPublicKeyFromPoint(p) + if err != nil { + return nil, fmt.Errorf("invalid K-256/secp256k1 public key: %w", err) + } + pub := PublicKeyK256{pubK256: pubK} + err = pub.ensureBytes() + if err != nil { + return nil, err + } + return &pub, nil +} + +// Loads a [PublicKeyK256] from raw bytes, as exported by the PublicKey.UncompressedBytes method. +// +// Calling code needs to know the key type ahead of time, and must remove any string encoding (hex encoding, base64, etc) before calling this function. +func ParsePublicUncompressedBytesK256(data []byte) (*PublicKeyK256, error) { + pubK, err := secp256k1secec.NewPublicKey(data) + if err != nil { + return nil, fmt.Errorf("invalid K-256/secp256k1 public key: %w", err) + } + pub := PublicKeyK256{pubK256: pubK} + err = pub.ensureBytes() + if err != nil { + return nil, err + } + return &pub, nil +} + +// Checks if the two public keys are the same. Note that the naive == operator does not work for most equality checks. +func (k *PublicKeyK256) Equal(other PublicKey) bool { + otherK256, ok := other.(*PublicKeyK256) + if ok { + return k.pubK256.Equal(otherK256.pubK256) + } + return false +} + +// verifies that this public key is safe to export as bytes later on +func (k *PublicKeyK256) ensureBytes() error { + p := k.pubK256.Point() + if p.IsIdentity() != 0 { + return fmt.Errorf("unexpected invalid K-256/secp256k1 public key (internal)") + } + return nil +} + +// Serializes the key in to "uncompressed" binary format. +func (k *PublicKeyK256) UncompressedBytes() []byte { + p := k.pubK256.Point() + return p.UncompressedBytes() +} + +// Serializes the key in to "compressed" binary format. +func (k *PublicKeyK256) Bytes() []byte { + p := k.pubK256.Point() + return p.CompressedBytes() +} + +// First hashes the raw bytes, then verifies the digest, returning `nil` for valid signatures, or an error for any failure. +// +// SHA-256 is the hash algorithm used, as specified by atproto. Signing digests is the norm for ECDSA, and required by some backend implementations. This method does not "double hash", it simply has name which clarifies that hashing is happening. +// +// Calling code is responsible for any string decoding of signatures (eg, hex or base64) before calling this function. +// +// This method requires a "low-S" signature, as specified by atproto. +func (k *PublicKeyK256) HashAndVerify(content, sig []byte) error { + hash := sha256.Sum256(content) + if !k.pubK256.Verify(hash[:], sig, k256Options) { + return ErrInvalidSignature + } + return nil +} + +// Same as HashAndVerify(), only does not require "low-S" signature. +// +// Used for, eg, JWT validation. +func (k *PublicKeyK256) HashAndVerifyLenient(content, sig []byte) error { + hash := sha256.Sum256(content) + if !k.pubK256.Verify(hash[:], sig, k256LenientOptions) { + return ErrInvalidSignature + } + return nil +} + +// Returns a multibased string encoding of the public key, including a multicodec indicator and compressed curve bytes serialization +func (k *PublicKeyK256) Multibase() string { + kbytes := k.Bytes() + // multicodec secp256k1-pub, code 0xE7, varint bytes: [0xE7, 0x01] + kbytes = append([]byte{0xE7, 0x01}, kbytes...) + return "z" + base58.Encode(kbytes) +} + +// Returns a did:key string encoding of the public key, as would be encoded in a DID PLC operation: +// +// - compressed / compacted binary representation +// - prefix with appropriate curve multicodec bytes +// - encode bytes with base58btc +// - add "z" prefix to indicate encoding +// - add "did:key:" prefix +func (k *PublicKeyK256) DIDKey() string { + return "did:key:" + k.Multibase() +} diff --git a/atproto/atcrypto/keys.go b/atproto/atcrypto/keys.go new file mode 100644 index 000000000..8e94a43f3 --- /dev/null +++ b/atproto/atcrypto/keys.go @@ -0,0 +1,142 @@ +package atcrypto + +import ( + "errors" + "fmt" + "strings" + + "github.com/mr-tron/base58" +) + +// Common interface for all the supported atproto cryptographic systems, when +// secret key material may not be directly available to be exported as bytes. +type PrivateKey interface { + Equal(other PrivateKey) bool + + PublicKey() (PublicKey, error) + + // Hashes the raw bytes using SHA-256, then signs the digest bytes. + // Always returns a "low-S" signature (for elliptic curve systems where that is ambiguous). + HashAndSign(content []byte) ([]byte, error) +} + +// Common interface for all the supported atproto cryptographic systems, when +// secret key material is directly available to be exported as bytes. +type PrivateKeyExportable interface { + PrivateKey + + // Untyped (no multicodec) encoding of the secret key material. + // The encoding format is curve-specific, and is generally "compact" for private keys. + // No ASN.1 or other enclosing structure is applied to the bytes. + Bytes() []byte + + // String serialization of the key bytes in "Multibase" format. + Multibase() string +} + +// Common interface for all the supported atproto cryptographic systems. +type PublicKey interface { + Equal(other PublicKey) bool + + // Compact byte serialization (for elliptic curve systems where encoding is ambiguous). + Bytes() []byte + + // Hashes the raw bytes using SHA-256, then verifies the signature of the digest bytes. + HashAndVerify(content, sig []byte) error + + // Same as HashAndVerify(), only does not require "low-S" signature. Used for, eg, JWT validation. + HashAndVerifyLenient(content, sig []byte) error + + // String serialization of the key bytes using common parameters: + // compressed byte serialization; multicode varint code prefix; base58btc + // string encoding ("z" prefix) + Multibase() string + + // String serialization of the key bytes as a did:key. + DIDKey() string + + // Non-compact byte serialization (for elliptic curve systems where + // encoding is ambiguous) + // + // This is not used frequently, or directly in atproto, but some + // serializations and encodings require it. + // + // For systems with no compressed/uncompressed distinction, returns the same + // value as Bytes(). + UncompressedBytes() []byte + + // Serialization as JWK struct (which can be marshalled to JSON) + JWK() (*JWK, error) +} + +var ErrInvalidSignature = errors.New("cryptographic signature invalid") + +/* +// quick code to verify varint byte conversion (https://play.golang.com/): +import ( + "encoding/binary" + "fmt" +) +buf := make([]byte, binary.MaxVarintLen64) +for _, x := range []uint64{0xE7, 0x1200, 0x1306, 0x1301} { + n := binary.PutUvarint(buf, x) + fmt.Printf("%x -> %x\n", x, buf[:n]) +} +*/ + +// Loads a private key from multibase string encoding, with multicodec indicating the key type. +func ParsePrivateMultibase(encoded string) (PrivateKeyExportable, error) { + if len(encoded) < 2 || encoded[0] != 'z' { + return nil, fmt.Errorf("crypto: not a multibase base58btc string") + } + data, err := base58.Decode(encoded[1:]) + if err != nil { + return nil, fmt.Errorf("crypto: not a multibase base58btc string") + } + if len(data) < 3 { + return nil, fmt.Errorf("crypto: multibase key was too short") + } + if data[0] == 0x86 && data[1] == 0x26 { + // multicodec p256-priv, code 0x1306, varint-encoded bytes: [0x86, 0x26] + return ParsePrivateBytesP256(data[2:]) + } else if data[0] == 0x81 && data[1] == 0x26 { + // multicodec secp256k1-priv, code 0x1301, varint-encoded bytes: [0x81, 0x26] + return ParsePrivateBytesK256(data[2:]) + } else { + return nil, fmt.Errorf("unsupported atproto key type (unknown multicodec prefix)") + } +} + +// Loads a public key from multibase string encoding, with multicodec indicating the key type. +func ParsePublicMultibase(encoded string) (PublicKey, error) { + if len(encoded) < 2 || encoded[0] != 'z' { + return nil, fmt.Errorf("crypto: not a multibase base58btc string") + } + data, err := base58.Decode(encoded[1:]) + if err != nil { + return nil, fmt.Errorf("crypto: not a multibase base58btc string") + } + if len(data) < 3 { + return nil, fmt.Errorf("crypto: multibase key was too short") + } + if data[0] == 0x80 && data[1] == 0x24 { + // multicodec p256-pub, code 0x1200, varint-encoded bytes: [0x80, 0x24] + return ParsePublicBytesP256(data[2:]) + } else if data[0] == 0xE7 && data[1] == 0x01 { + // multicodec secp256k1-pub, code 0xE7, varint bytes: [0xE7, 0x01] + return ParsePublicBytesK256(data[2:]) + } else { + return nil, fmt.Errorf("unsupported atproto key type (unknown multicodec prefix)") + } +} + +// Loads a [PublicKey] from did:key string serialization. +// +// The did:key format encodes the key type. +func ParsePublicDIDKey(didKey string) (PublicKey, error) { + if !strings.HasPrefix(didKey, "did:key:z") { + return nil, fmt.Errorf("string is not a DID key: %s", didKey) + } + mb := strings.TrimPrefix(didKey, "did:key:") + return ParsePublicMultibase(mb) +} diff --git a/atproto/atcrypto/keys_test.go b/atproto/atcrypto/keys_test.go new file mode 100644 index 000000000..80909041a --- /dev/null +++ b/atproto/atcrypto/keys_test.go @@ -0,0 +1,176 @@ +package atcrypto + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKeyBasics(t *testing.T) { + assert := assert.New(t) + + // try signing/verifying a couple different message sizes. these all just get hashed. + msg := []byte("test-message") + midMsg := make([]byte, 13*1024) + _, err := rand.Read(midMsg) + assert.NoError(err) + bigMsg := make([]byte, 16*1024*1024) + _, err = rand.Read(bigMsg) + assert.NoError(err) + + // private key generation and byte serialization (P-256) + privP256, err := GeneratePrivateKeyP256() + assert.NoError(err) + privP256Bytes := privP256.Bytes() + privP256FromBytes, err := ParsePrivateBytesP256(privP256Bytes) + assert.NoError(err) + assert.True(privP256.Equal(privP256FromBytes)) + privP256MB := privP256.Multibase() + privP256FromMB, err := ParsePrivateMultibase(privP256MB) + assert.NoError(err) + assert.True(privP256.Equal(privP256FromMB)) + + // private key generation and byte serialization (K-256) + privK256, err := GeneratePrivateKeyK256() + assert.NoError(err) + privK256Bytes := privK256.Bytes() + privK256FromBytes, err := ParsePrivateBytesK256(privK256Bytes) + assert.NoError(err) + assert.Equal(privK256, privK256FromBytes) + privK256MB := privK256.Multibase() + privK256FromMB, err := ParsePrivateMultibase(privK256MB) + assert.NoError(err) + assert.True(privK256.Equal(privK256FromMB)) + + // public key byte serialization (P-256) + pubP256, err := privP256.PublicKey() + assert.NoError(err) + pubP256CompBytes := pubP256.Bytes() + pubP256FromCompBytes, err := ParsePublicBytesP256(pubP256CompBytes) + assert.NoError(err) + assert.True(pubP256.Equal(pubP256FromCompBytes)) + + pubP256UncompBytes := pubP256.UncompressedBytes() + pubP256FromUncompBytes, err := ParsePublicUncompressedBytesP256(pubP256UncompBytes) + assert.NoError(err) + assert.True(pubP256.Equal(pubP256FromUncompBytes)) + + both := []PrivateKey{privP256, privK256} + for _, priv := range both { + pub, err := priv.PublicKey() + assert.NoError(err) + + // public key encoding + pubDIDKeyString := pub.DIDKey() + pubDK, err := ParsePublicDIDKey(pubDIDKeyString) + assert.NoError(err) + assert.True(pub.Equal(pubDK)) + + pubMultibaseString := pub.Multibase() + pubMB, err := ParsePublicMultibase(pubMultibaseString) + assert.NoError(err) + assert.True(pub.Equal(pubMB)) + + // signature verification + sig, err := priv.HashAndSign(msg) + assert.NoError(err) + assert.NoError(pub.HashAndVerify(msg, sig)) + + midSig, err := priv.HashAndSign(midMsg) + assert.NoError(err) + assert.NoError(pub.HashAndVerify(midMsg, midSig)) + + bigSig, err := priv.HashAndSign(bigMsg) + assert.NoError(err) + assert.NoError(pub.HashAndVerify(bigMsg, bigSig)) + } +} + +// this does a large number of sign/verify cycles, to try and hit any bad high-S signatures +func TestLowSMany(t *testing.T) { + assert := assert.New(t) + + msg := make([]byte, 1024) + + for i := 0; i < 128; i++ { + privP256, err := GeneratePrivateKeyP256() + assert.NoError(err) + privK256, err := GeneratePrivateKeyK256() + assert.NoError(err) + + both := []PrivateKey{privP256, privK256} + for _, priv := range both { + pub, err := priv.PublicKey() + assert.NoError(err) + + _, err = rand.Read(msg) + assert.NoError(err) + + sig, err := priv.HashAndSign(msg) + assert.NoError(err) + err = pub.HashAndVerify(msg, sig) + assert.NoError(err) + // bail out early instead of looping + if err != nil { + break + } + } + } +} + +func TestKeyCompressionP256(t *testing.T) { + assert := assert.New(t) + + priv, err := GeneratePrivateKeyP256() + assert.NoError(err) + privBytes := priv.Bytes() + pub, err := priv.PublicKey() + assert.NoError(err) + sig, err := priv.HashAndSign([]byte("test-message")) + assert.NoError(err) + + // P-256 key and signature sizes + assert.Equal(32, len(privBytes)) + assert.Equal(33, len(pub.Bytes())) + assert.Equal(65, len(pub.UncompressedBytes())) + assert.Equal(64, len(sig)) +} + +func TestKeyCompressionK256(t *testing.T) { + assert := assert.New(t) + + priv, err := GeneratePrivateKeyK256() + assert.NoError(err) + privBytes := priv.Bytes() + pub, err := priv.PublicKey() + assert.NoError(err) + sig, err := priv.HashAndSign([]byte("test-message")) + assert.NoError(err) + + // K-256 key and signature sizes + assert.Equal(32, len(privBytes)) + assert.Equal(33, len(pub.Bytes())) + assert.Equal(65, len(pub.UncompressedBytes())) + assert.Equal(64, len(sig)) +} + +func TestParsePrivateMultibase(t *testing.T) { + assert := assert.New(t) + + // these values generated by this library, not an external source + privP256MB := "z42tvqQS5sVhaV1jLZ5P6ZKEPEbSpYavNVmT88YDYV3MEZ8D" + privK256MB := "z3vLWgA9nXoPzxsJJafDY9BPrZd3EDWjvcCtYfrFxZ7xbMVi" + + privP256FromMB, err := ParsePrivateMultibase(privP256MB) + assert.NoError(err) + _, ok := privP256FromMB.(*PrivateKeyP256) + assert.True(ok) + assert.Equal(privP256MB, privP256FromMB.Multibase()) + + privK256FromMB, err := ParsePrivateMultibase(privK256MB) + assert.NoError(err) + _, ok = privK256FromMB.(*PrivateKeyK256) + assert.True(ok) + assert.Equal(privK256MB, privK256FromMB.Multibase()) +} diff --git a/atproto/atcrypto/p256.go b/atproto/atcrypto/p256.go new file mode 100644 index 000000000..09ad51dff --- /dev/null +++ b/atproto/atcrypto/p256.go @@ -0,0 +1,267 @@ +package atcrypto + +import ( + "crypto/ecdh" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "fmt" + "math/big" + + "github.com/mr-tron/base58" +) + +// Implements the [PrivateKeyExportable] and [PrivateKey] interfaces for the NIST P-256 / secp256r1 / ES256 cryptographic curve. +// Secret key material is naively stored in memory. +type PrivateKeyP256 struct { + privP256ecdh *ecdh.PrivateKey + privP256 ecdsa.PrivateKey +} + +// Implements the [PublicKey] interface for the NIST P-256 / secp256r1 / ES256 cryptographic curve. +type PublicKeyP256 struct { + pubP256 ecdsa.PublicKey +} + +var _ PrivateKey = (*PrivateKeyP256)(nil) +var _ PrivateKeyExportable = (*PrivateKeyP256)(nil) +var _ PublicKey = (*PublicKeyP256)(nil) + +// Creates a secure new cryptographic key from scratch. +func GeneratePrivateKeyP256() (*PrivateKeyP256, error) { + skECDSA, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("P-256/secp256r1 key generation failed: %w", err) + } + skECDH, err := skECDSA.ECDH() + if err != nil { + return nil, fmt.Errorf("unexpected internal error converting P-256 key from ecdsa to ecdh: %w", err) + } + return &PrivateKeyP256{privP256: *skECDSA, privP256ecdh: skECDH}, nil +} + +// Loads a [PrivateKeyP256] from raw bytes, as exported by the PrivateKeyP256.Bytes method. +// +// Calling code needs to know the key type ahead of time, and must remove any string encoding (hex encoding, base64, etc) before calling this function. +func ParsePrivateBytesP256(data []byte) (*PrivateKeyP256, error) { + // elaborately parse as an ecdh.PrivateKey, then get from that to ecdsa.PrivateKey by encoding/decoding using x509 PKCS8 encoding. + // Note that the 'data' bytes format is *not* x509 PKCS8! + skECDH, err := ecdh.P256().NewPrivateKey(data) + if err != nil { + return nil, fmt.Errorf("invalid P-256/secp256r1 private key: %w", err) + } + enc, err := x509.MarshalPKCS8PrivateKey(skECDH) + if err != nil { + return nil, fmt.Errorf("invalid P-256/secp256r1 private key: %w", err) + } + sk, err := x509.ParsePKCS8PrivateKey(enc) + if err != nil { + return nil, fmt.Errorf("invalid P-256/secp256r1 private key: %w", err) + } + skECDSA, ok := sk.(*ecdsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("unexpected internal error parsing own private P-256 x509 key: %w", err) + } + return &PrivateKeyP256{privP256: *skECDSA, privP256ecdh: skECDH}, nil +} + +// Checks if the two private keys are the same. Note that the naive == operator does not work for most equality checks. +func (k *PrivateKeyP256) Equal(other PrivateKey) bool { + otherP256, ok := other.(*PrivateKeyP256) + if ok { + return k.privP256.Equal(&otherP256.privP256) + } + return false +} + +// Serializes the secret key material in to a raw binary format, which can be parsed by [ParsePrivateBytesP256]. +// +// For P-256, this is the "compact" encoding and is 32 bytes long. There is no ASN.1 or other enclosing structure. +func (k *PrivateKeyP256) Bytes() []byte { + return k.privP256ecdh.Bytes() +} + +// Multibase string encoding of the private key, including a multicodec indicator +func (k *PrivateKeyP256) Multibase() string { + kbytes := k.Bytes() + // multicodec p256-priv, code 0x1306, varint-encoded bytes: [0x86, 0x26] + kbytes = append([]byte{0x86, 0x26}, kbytes...) + return "z" + base58.Encode(kbytes) +} + +// Outputs the [PublicKey] corresponding to this [PrivateKeyP256]; it will be a [PublicKeyP256]. +func (k *PrivateKeyP256) PublicKey() (PublicKey, error) { + pkECDSA, ok := k.privP256.Public().(*ecdsa.PublicKey) + if !ok { + return nil, fmt.Errorf("unexpected internal error casting P-256 ecdsa public key") + } + return &PublicKeyP256{pubP256: *pkECDSA}, nil +} + +// First hashes the raw bytes, then signs the digest, returning a binary signature. +// +// SHA-256 is the hash algorithm used, as specified by atproto. Signing digests is the norm for ECDSA, and required by some backend implementations. This method does not "double hash", it simply has name which clarifies that hashing is happening. +// +// Calling code is responsible for any string encoding of signatures (eg, hex or base64). For P-256, the signature is 64 bytes long. +// +// NIST ECDSA signatures can have a "malleability" issue, meaning that there are multiple valid signatures for the same content with the same signing key. This method always returns a "low-S" signature, as required by atproto. +func (k *PrivateKeyP256) HashAndSign(content []byte) ([]byte, error) { + hash := sha256.Sum256(content) + r, s, err := ecdsa.Sign(rand.Reader, &k.privP256, hash[:]) + if err != nil { + return nil, fmt.Errorf("crypto error signing with P-256/secp256r1 private key: %w", err) + } + s = sigSToLowS_P256(s) + sig := make([]byte, 64) + r.FillBytes(sig[:32]) + s.FillBytes(sig[32:]) + return sig, nil +} + +// Loads a [PublicKeyP256] raw bytes, as exported by the PublicKey.Bytes method. This is the "compressed" curve format. +// +// Calling code needs to know the key type ahead of time, and must remove any string encoding (hex encoding, base64, etc) before calling this function. +func ParsePublicBytesP256(data []byte) (*PublicKeyP256, error) { + curve := elliptic.P256() + x, y := elliptic.UnmarshalCompressed(curve, data) + if x == nil { + return nil, fmt.Errorf("invalid P-256 public key (x==nil)") + } + if !curve.Params().IsOnCurve(x, y) { + return nil, fmt.Errorf("invalid P-256 public key (not on curve)") + } + pubECDSA := &ecdsa.PublicKey{ + Curve: curve, + X: x, + Y: y, + } + pub := PublicKeyP256{pubP256: *pubECDSA} + err := pub.checkCurve() + if err != nil { + return nil, err + } + return &pub, nil +} + +// Loads a [PublicKeyP256] from raw bytes, as exported by the PublicKey.UncompressedBytes method. +// +// Calling code needs to know the key type ahead of time, and must remove any string encoding (hex encoding, base64, etc) before calling this function. +func ParsePublicUncompressedBytesP256(data []byte) (*PublicKeyP256, error) { + curve := elliptic.P256() + x, y := elliptic.Unmarshal(curve, data) + if x == nil { + return nil, fmt.Errorf("invalid P-256 public key (x==nil)") + } + if !curve.Params().IsOnCurve(x, y) { + return nil, fmt.Errorf("invalid P-256 public key (not on curve)") + } + pubECDSA := &ecdsa.PublicKey{ + Curve: curve, + X: x, + Y: y, + } + pub := PublicKeyP256{pubP256: *pubECDSA} + err := pub.checkCurve() + if err != nil { + return nil, err + } + return &pub, nil +} + +// Checks if the two public keys are the same. Note that the naive == operator does not work for most equality checks. +func (k *PublicKeyP256) Equal(other PublicKey) bool { + otherP256, ok := other.(*PublicKeyP256) + if ok { + return k.pubP256.Equal(&otherP256.pubP256) + } + return false +} + +func (k *PublicKeyP256) checkCurve() error { + if !k.pubP256.Curve.IsOnCurve(k.pubP256.X, k.pubP256.Y) { + return fmt.Errorf("unexpected invalid P-256/secp256r1 public key (internal)") + } + return nil +} + +// Serializes the key in to "uncompressed" binary format. +func (k *PublicKeyP256) UncompressedBytes() []byte { + return elliptic.Marshal(k.pubP256.Curve, k.pubP256.X, k.pubP256.Y) +} + +// Serializes the key in to "compressed" binary format. +func (k *PublicKeyP256) Bytes() []byte { + return elliptic.MarshalCompressed(k.pubP256.Curve, k.pubP256.X, k.pubP256.Y) +} + +// Hashes the raw bytes using SHA-256, then verifies the signature against the digest bytes. +// +// Signing digests is the norm for ECDSA, and required by some backend implementations. This method does not "double hash", it simply has name which clarifies that hashing is happening. +// +// Calling code is responsible for any string decoding of signatures (eg, hex or base64) before calling this function. +// +// This method requires a "low-S" signature, as specified by atproto. +func (k *PublicKeyP256) HashAndVerify(content, sig []byte) error { + hash := sha256.Sum256(content) + // parseP256Sig + if len(sig) != 64 { + return fmt.Errorf("crypto: P-256 signatures must be 64 bytes, got len=%d", len(sig)) + } + r := big.NewInt(0) + s := big.NewInt(0) + r.SetBytes(sig[:32]) + s.SetBytes(sig[32:]) + + if !ecdsa.Verify(&k.pubP256, hash[:], r, s) { + return ErrInvalidSignature + } + + // ensure that signature is low-S + if !sigSIsLowS_P256(s) { + return ErrInvalidSignature + } + + return nil +} + +// Same as HashAndVerify(), only does not require "low-S" signature. +// +// Used for, eg, JWT validation. +func (k *PublicKeyP256) HashAndVerifyLenient(content, sig []byte) error { + hash := sha256.Sum256(content) + // parseP256Sig + if len(sig) != 64 { + return fmt.Errorf("crypto: P-256 signatures must be 64 bytes, got len=%d", len(sig)) + } + r := big.NewInt(0) + s := big.NewInt(0) + r.SetBytes(sig[:32]) + s.SetBytes(sig[32:]) + + if !ecdsa.Verify(&k.pubP256, hash[:], r, s) { + return ErrInvalidSignature + } + + return nil +} + +// Multibase string encoding of the public key, including a multicodec indicator and compressed curve bytes serialization +func (k *PublicKeyP256) Multibase() string { + kbytes := k.Bytes() + // multicodec p256-pub, code 0x1200, varint-encoded bytes: [0x80, 0x24] + kbytes = append([]byte{0x80, 0x24}, kbytes...) + return "z" + base58.Encode(kbytes) +} + +// did:key string encoding of the public key, as would be encoded in a DID PLC operation: +// +// - compressed / compacted binary representation +// - prefix with appropriate curve multicodec bytes +// - encode bytes with base58btc +// - add "z" prefix to indicate encoding +// - add "did:key:" prefix +func (k *PublicKeyP256) DIDKey() string { + return "did:key:" + k.Multibase() +} diff --git a/atproto/atcrypto/p256_lowS.go b/atproto/atcrypto/p256_lowS.go new file mode 100644 index 000000000..9d5b6e058 --- /dev/null +++ b/atproto/atcrypto/p256_lowS.go @@ -0,0 +1,28 @@ +package atcrypto + +import ( + "crypto/elliptic" + "math/big" +) + +var curveN_P256 *big.Int = elliptic.P256().Params().N +var curveHalfOrder_P256 *big.Int = new(big.Int).Rsh(curveN_P256, 1) + +// Checks if 'S' value from a P-256 signature is "low-S". +// un-reviewed, un-safe code from: https://github.com/golang/go/issues/54549 +func sigSIsLowS_P256(s *big.Int) bool { + return s.Cmp(curveHalfOrder_P256) != 1 +} + +// Ensures that 'S' value from a P-256 signature is "low-S" variant. +// un-reviewed, un-safe code from: https://github.com/golang/go/issues/54549 +func sigSToLowS_P256(s *big.Int) *big.Int { + + if !sigSIsLowS_P256(s) { + // Set s to N - s that will be then in the lower part of signature space + // less or equal to half order + s.Sub(curveN_P256, s) + return s + } + return s +} diff --git a/atproto/atcrypto/testdata/signature-fixtures.json b/atproto/atcrypto/testdata/signature-fixtures.json new file mode 100644 index 000000000..2e41be58c --- /dev/null +++ b/atproto/atcrypto/testdata/signature-fixtures.json @@ -0,0 +1,68 @@ +[ + { + "comment": "valid P-256 key and signature, with low-S signature", + "messageBase64": "oWVoZWxsb2V3b3JsZA", + "algorithm": "ES256", + "didDocSuite": "EcdsaSecp256r1VerificationKey2019", + "publicKeyDid": "did:key:zDnaembgSGUhZULN2Caob4HLJPaxBh92N7rtH21TErzqf8HQo", + "publicKeyMultibase": "zxdM8dSstjrpZaRUwBmDvjGXweKuEMVN95A9oJBFjkWMh", + "signatureBase64": "2vZNsG3UKvvO/CDlrdvyZRISOFylinBh0Jupc6KcWoJWExHptCfduPleDbG3rko3YZnn9Lw0IjpixVmexJDegg", + "validSignature": true, + "tags": [] + }, + { + "comment": "valid K-256 key and signature, with low-S signature", + "messageBase64": "oWVoZWxsb2V3b3JsZA", + "algorithm": "ES256K", + "didDocSuite": "EcdsaSecp256k1VerificationKey2019", + "publicKeyDid": "did:key:zQ3shqwJEJyMBsBXCWyCBpUBMqxcon9oHB7mCvx4sSpMdLJwc", + "publicKeyMultibase": "z25z9DTpsiYYJKGsWmSPJK2NFN8PcJtZig12K59UgW7q5t", + "signatureBase64": "5WpdIuEUUfVUYaozsi8G0B3cWO09cgZbIIwg1t2YKdUn/FEznOndsz/qgiYb89zwxYCbB71f7yQK5Lr7NasfoA", + "validSignature": true, + "tags": [] + }, + { + "comment": "P-256 key and signature, with non-low-S signature which is invalid in atproto", + "messageBase64": "oWVoZWxsb2V3b3JsZA", + "algorithm": "ES256", + "didDocSuite": "EcdsaSecp256r1VerificationKey2019", + "publicKeyDid": "did:key:zDnaembgSGUhZULN2Caob4HLJPaxBh92N7rtH21TErzqf8HQo", + "publicKeyMultibase": "zxdM8dSstjrpZaRUwBmDvjGXweKuEMVN95A9oJBFjkWMh", + "signatureBase64": "2vZNsG3UKvvO/CDlrdvyZRISOFylinBh0Jupc6KcWoKp7O4VS9giSAah8k5IUbXIW00SuOrjfEqQ9HEkN9JGzw", + "validSignature": false, + "tags": ["high-s"] + }, + { + "comment": "K-256 key and signature, with non-low-S signature which is invalid in atproto", + "messageBase64": "oWVoZWxsb2V3b3JsZA", + "algorithm": "ES256K", + "didDocSuite": "EcdsaSecp256k1VerificationKey2019", + "publicKeyDid": "did:key:zQ3shqwJEJyMBsBXCWyCBpUBMqxcon9oHB7mCvx4sSpMdLJwc", + "publicKeyMultibase": "z25z9DTpsiYYJKGsWmSPJK2NFN8PcJtZig12K59UgW7q5t", + "signatureBase64": "5WpdIuEUUfVUYaozsi8G0B3cWO09cgZbIIwg1t2YKdXYA67MYxYiTMAVfdnkDCMN9S5B3vHosRe07aORmoshoQ", + "validSignature": false, + "tags": ["high-s"] + }, + { + "comment": "P-256 key and signature, with DER-encoded signature which is invalid in atproto", + "messageBase64": "oWVoZWxsb2V3b3JsZA", + "algorithm": "ES256", + "didDocSuite": "EcdsaSecp256r1VerificationKey2019", + "publicKeyDid": "did:key:zDnaeT6hL2RnTdUhAPLij1QBkhYZnmuKyM7puQLW1tkF4Zkt8", + "publicKeyMultibase": "ze8N2PPxnu19hmBQ58t5P3E9Yj6CqakJmTVCaKvf9Byq2", + "signatureBase64": "MEQCIFxYelWJ9lNcAVt+jK0y/T+DC/X4ohFZ+m8f9SEItkY1AiACX7eXz5sgtaRrz/SdPR8kprnbHMQVde0T2R8yOTBweA", + "validSignature": false, + "tags": ["der-encoded"] + }, + { + "comment": "K-256 key and signature, with DER-encoded signature which is invalid in atproto", + "messageBase64": "oWVoZWxsb2V3b3JsZA", + "algorithm": "ES256K", + "didDocSuite": "EcdsaSecp256k1VerificationKey2019", + "publicKeyDid": "did:key:zQ3shnriYMXc8wvkbJqfNWh5GXn2bVAeqTC92YuNbek4npqGF", + "publicKeyMultibase": "z22uZXWP8fdHXi4jyx8cCDiBf9qQTsAe6VcycoMQPfcMQX", + "signatureBase64": "MEUCIQCWumUqJqOCqInXF7AzhIRg2MhwRz2rWZcOEsOjPmNItgIgXJH7RnqfYY6M0eg33wU0sFYDlprwdOcpRn78Sz5ePgk", + "validSignature": false, + "tags": ["der-encoded"] + } +] diff --git a/atproto/atcrypto/testdata/w3c_didkey_K256.json b/atproto/atcrypto/testdata/w3c_didkey_K256.json new file mode 100644 index 000000000..9ba9844b3 --- /dev/null +++ b/atproto/atcrypto/testdata/w3c_didkey_K256.json @@ -0,0 +1,22 @@ +[ + { + "privateKeyBytesHex": "9085d2bef69286a6cbb51623c8fa258629945cd55ca705cc4e66700396894e0c", + "publicDidKey": "did:key:zQ3shokFTS3brHcDQrn82RUDfCZESWL1ZdCEJwekUDPQiYBme" + }, + { + "privateKeyBytesHex": "f0f4df55a2b3ff13051ea814a8f24ad00f2e469af73c363ac7e9fb999a9072ed", + "publicDidKey": "did:key:zQ3shtxV1FrJfhqE1dvxYRcCknWNjHc3c5X1y3ZSoPDi2aur2" + }, + { + "privateKeyBytesHex": "6b0b91287ae3348f8c2f2552d766f30e3604867e34adc37ccbb74a8e6b893e02", + "publicDidKey": "did:key:zQ3shZc2QzApp2oymGvQbzP8eKheVshBHbU4ZYjeXqwSKEn6N" + }, + { + "privateKeyBytesHex": "c0a6a7c560d37d7ba81ecee9543721ff48fea3e0fb827d42c1868226540fac15", + "publicDidKey": "did:key:zQ3shadCps5JLAHcZiuX5YUtWHHL8ysBJqFLWvjZDKAWUBGzy" + }, + { + "privateKeyBytesHex": "175a232d440be1e0788f25488a73d9416c04b6f924bea6354bf05dd2f1a75133", + "publicDidKey": "did:key:zQ3shptjE6JwdkeKN4fcpnYQY3m9Cet3NiHdAfpvSUZBFoKBj" + } +] diff --git a/atproto/atcrypto/testdata/w3c_didkey_P256.json b/atproto/atcrypto/testdata/w3c_didkey_P256.json new file mode 100644 index 000000000..65d50933a --- /dev/null +++ b/atproto/atcrypto/testdata/w3c_didkey_P256.json @@ -0,0 +1,6 @@ +[ + { + "privateKeyBytesBase58": "9p4VRzdmhsnq869vQjVCTrRry7u4TtfRxhvBFJTGU2Cp", + "publicDidKey": "did:key:zDnaeTiq1PdzvZXUaMdezchcMJQpBdH2VN4pgrrEhMCCbmwSb" + } +] diff --git a/atproto/atcrypto/w3c_didkey_test.go b/atproto/atcrypto/w3c_didkey_test.go new file mode 100644 index 000000000..951e0b203 --- /dev/null +++ b/atproto/atcrypto/w3c_didkey_test.go @@ -0,0 +1,96 @@ +package atcrypto + +import ( + "encoding/hex" + "encoding/json" + "io" + "os" + "testing" + + "github.com/mr-tron/base58" + "github.com/stretchr/testify/assert" +) + +type DIDKeyFixture struct { + PrivateKeyBytesBase58 string `json:"privateKeyBytesBase58"` + PrivateKeyBytesHex string `json:"privateKeyBytesHex"` + PublicDIDKey string `json:"publicDidKey"` +} + +func TestDIDKeyFixtures(t *testing.T) { + + fixtureBatches := []struct { + path string + keyType string + }{ + {path: "testdata/w3c_didkey_P256.json", keyType: "P256"}, + {path: "testdata/w3c_didkey_K256.json", keyType: "K256"}, + } + + for _, batch := range fixtureBatches { + + f, err := os.Open(batch.path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + fixBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []DIDKeyFixture + if err := json.Unmarshal(fixBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, row := range fixtures { + testDIDKeyFixture(t, row, batch.keyType) + } + } +} + +func testDIDKeyFixture(t *testing.T, row DIDKeyFixture, keyType string) { + assert := assert.New(t) + + var raw []byte + var err error + if row.PrivateKeyBytesBase58 != "" { + raw, err = base58.Decode(row.PrivateKeyBytesBase58) + if err != nil { + t.Fatal(err) + } + } else if row.PrivateKeyBytesHex != "" { + raw, err = hex.DecodeString(row.PrivateKeyBytesHex) + if err != nil { + t.Fatal(err) + } + } else { + t.Fatal("no private key found") + } + + var priv PrivateKey + switch keyType { + case "P256": + priv, err = ParsePrivateBytesP256(raw) + case "K256": + priv, err = ParsePrivateBytesK256(raw) + default: + t.Fatal("impossible key type") + } + if err != nil { + t.Fatal(err) + } + kBytes, err := priv.PublicKey() + if err != nil { + t.Fatal(err) + } + kDIDKey, err := ParsePublicDIDKey(row.PublicDIDKey) + if err != nil { + t.Fatal(err) + } + assert.True(kBytes.Equal(kDIDKey)) + assert.Equal(row.PublicDIDKey, kBytes.DIDKey()) + assert.Equal(row.PublicDIDKey, kDIDKey.DIDKey()) +} diff --git a/atproto/atdata/basic_test.go b/atproto/atdata/basic_test.go new file mode 100644 index 000000000..bae3d88f0 --- /dev/null +++ b/atproto/atdata/basic_test.go @@ -0,0 +1,87 @@ +package atdata + +import ( + "encoding/json" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/assert" +) + +func TestSimpleValidation(t *testing.T) { + assert := assert.New(t) + + s := "a string" + assert.NoError(Validate(map[string]interface{}{ + "a": 5, + "b": 123, + "c": s, + "d": &s, + })) + assert.NoError(Validate(map[string]interface{}{ + "$type": "com.example.thing", + "a": 5, + })) + assert.Error(Validate(map[string]interface{}{ + "$type": 123, + "a": 5, + })) + assert.Error(Validate(map[string]interface{}{ + "$type": "", + "a": 5, + })) +} + +func TestSyntaxSerialize(t *testing.T) { + assert := assert.New(t) + + atid, err := syntax.ParseAtIdentifier("did:web:example.com") + assert.NoError(err) + obj := map[string]interface{}{ + "at-identifier": atid, + "at-uri": syntax.ATURI("at://did:abc:123/io.nsid.someFunc/record-key"), + "cid-string": syntax.CID("bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"), + "datetime": syntax.Datetime("2023-10-30T22:25:23Z"), + "did": syntax.DID("did:web:example.com"), + "handle": syntax.Handle("blah.example.com"), + "language": syntax.Language("us"), + "nsid": syntax.NSID("com.example.blah"), + "recordkey": syntax.RecordKey("self"), + "tid": syntax.TID("3kao2cl6lyj2p"), + "uri": syntax.URI("https://example.com/file"), + } + assert.NoError(Validate(obj)) + _, err = MarshalCBOR(obj) + assert.NoError(err) + _, err = json.Marshal(obj) + assert.NoError(err) +} + +func TestExtractBlobs(t *testing.T) { + assert := assert.New(t) + + cid1, _ := cid.Parse("bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity") + obj := map[string]interface{}{ + "a": 5, + "b": 123, + "c": map[string]interface{}{ + "blb": Blob{ + Size: 567, + MimeType: "image/jpeg", + Ref: CIDLink(cid1), + }, + }, + "d": []interface{}{ + 123, + Blob{ + Size: 123, + MimeType: "image/png", + Ref: CIDLink(cid1), + }, + }, + } + blbs := ExtractBlobs(obj) + assert.Equal(2, len(blbs)) +} diff --git a/atproto/atdata/blob.go b/atproto/atdata/blob.go new file mode 100644 index 000000000..3537e27cd --- /dev/null +++ b/atproto/atdata/blob.go @@ -0,0 +1,146 @@ +package atdata + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" +) + +// Represents the "blob" type from the atproto data model. +// +// This struct does not get marshaled/unmarshaled directly in to JSON or CBOR; see the BlobSchema and LegacyBlobSchema structs. This is the type that should be included in golang struct definitions. +// +// When representing a "legacy" blob (no size field, string CID), size == -1. +type Blob struct { + Ref CIDLink + MimeType string + Size int64 +} + +type LegacyBlobSchema struct { + Cid string `json:"cid" cborgen:"cid"` + MimeType string `json:"mimeType" cborgen:"mimeType"` +} + +type BlobSchema struct { + LexiconTypeID string `json:"$type" cborgen:"$type,const=blob"` + Ref CIDLink `json:"ref" cborgen:"ref"` + MimeType string `json:"mimeType" cborgen:"mimeType"` + Size int64 `json:"size" cborgen:"size"` +} + +func (b Blob) MarshalJSON() ([]byte, error) { + if b.Size < 0 { + lb := LegacyBlobSchema{ + Cid: b.Ref.String(), + MimeType: b.MimeType, + } + return json.Marshal(lb) + } else { + nb := BlobSchema{ + LexiconTypeID: "blob", + Ref: b.Ref, + MimeType: b.MimeType, + Size: b.Size, + } + return json.Marshal(nb) + } +} + +func (b *Blob) UnmarshalJSON(raw []byte) error { + typ, err := ExtractTypeJSON(raw) + if err != nil { + return fmt.Errorf("parsing blob type: %v", err) + } + + if typ == "blob" { + var bs BlobSchema + err := json.Unmarshal(raw, &bs) + if err != nil { + return fmt.Errorf("parsing blob JSON: %v", err) + } + b.Ref = bs.Ref + b.MimeType = bs.MimeType + b.Size = bs.Size + if bs.Size < 0 { + return fmt.Errorf("parsing blob: negative size: %d", bs.Size) + } + } else { + var legacy LegacyBlobSchema + err := json.Unmarshal(raw, &legacy) + if err != nil { + return fmt.Errorf("parsing legacy blob: %v", err) + } + refCid, err := cid.Decode(legacy.Cid) + if err != nil { + return fmt.Errorf("parsing CID in legacy blob: %v", err) + } + b.Ref = CIDLink(refCid) + b.MimeType = legacy.MimeType + b.Size = -1 + } + return nil +} + +func (b *Blob) MarshalCBOR(w io.Writer) error { + if b == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if b.Size < 0 { + lb := LegacyBlobSchema{ + Cid: b.Ref.String(), + MimeType: b.MimeType, + } + return lb.MarshalCBOR(w) + } else { + bs := BlobSchema{ + LexiconTypeID: "blob", + Ref: b.Ref, + MimeType: b.MimeType, + Size: b.Size, + } + return bs.MarshalCBOR(w) + } +} + +func (lb *Blob) UnmarshalCBOR(r io.Reader) error { + typ, b, err := ExtractTypeCBORReader(r) + if err != nil { + return fmt.Errorf("parsing $blob CBOR type: %w", err) + } + + *lb = Blob{} + if typ == "blob" { + var bs BlobSchema + err := bs.UnmarshalCBOR(bytes.NewReader(b)) + if err != nil { + return fmt.Errorf("parsing $blob CBOR: %v", err) + } + lb.Ref = bs.Ref + lb.MimeType = bs.MimeType + lb.Size = bs.Size + if bs.Size < 0 { + return fmt.Errorf("parsing $blob CBOR: negative size: %d", bs.Size) + } + } else { + legacy := LegacyBlobSchema{} + err := legacy.UnmarshalCBOR(bytes.NewReader(b)) + if err != nil { + return fmt.Errorf("parsing legacy blob CBOR: %v", err) + } + refCid, err := cid.Decode(legacy.Cid) + if err != nil { + return fmt.Errorf("parsing CID in legacy blob CBOR: %v", err) + } + lb.Ref = CIDLink(refCid) + lb.MimeType = legacy.MimeType + lb.Size = -1 + } + + return nil +} diff --git a/atproto/atdata/bytes.go b/atproto/atdata/bytes.go new file mode 100644 index 000000000..dcee4aedc --- /dev/null +++ b/atproto/atdata/bytes.go @@ -0,0 +1,67 @@ +package atdata + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + + cbg "github.com/whyrusleeping/cbor-gen" +) + +// Represents the "bytes" type from the atproto data model. +// +// In JSON, marshals to an object with $bytes key and base64-encoded data. +// +// In CBOR, marshals to a byte array. +type Bytes []byte + +type JsonBytes struct { + Bytes string `json:"$bytes"` +} + +func (lb Bytes) MarshalJSON() ([]byte, error) { + if lb == nil { + return nil, fmt.Errorf("tried to marshal nil $bytes") + } + jb := JsonBytes{ + Bytes: base64.RawStdEncoding.EncodeToString([]byte(lb)), + } + return json.Marshal(jb) +} + +func (lb *Bytes) UnmarshalJSON(raw []byte) error { + var jb JsonBytes + err := json.Unmarshal(raw, &jb) + if err != nil { + return fmt.Errorf("parsing $bytes JSON: %v", err) + } + out, err := base64.RawStdEncoding.DecodeString(jb.Bytes) + if err != nil { + return fmt.Errorf("parsing $bytes base64: %v", err) + } + *lb = Bytes(out) + return nil +} + +func (lb *Bytes) MarshalCBOR(w io.Writer) error { + if lb == nil { + _, err := w.Write(cbg.CborNull) + return err + } + cw := cbg.NewCborWriter(w) + if err := cbg.WriteByteArray(cw, ([]byte)(*lb)); err != nil { + return fmt.Errorf("failed to write $bytes as CBOR: %w", err) + } + return nil +} + +func (lb *Bytes) UnmarshalCBOR(r io.Reader) error { + cr := cbg.NewCborReader(r) + b, err := cbg.ReadByteArray(cr, MAX_RECORD_BYTES_LEN) + if err != nil { + return fmt.Errorf("failed to read $bytes from CBOR: %w", err) + } + *lb = Bytes(b) + return nil +} diff --git a/atproto/atdata/cbor_gen.go b/atproto/atdata/cbor_gen.go new file mode 100644 index 000000000..a499d6ad9 --- /dev/null +++ b/atproto/atdata/cbor_gen.go @@ -0,0 +1,458 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package atdata + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *GenericRecord) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Type (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if len(t.Type) > 1000000 { + return xerrors.Errorf("Value in field t.Type was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Type))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Type)); err != nil { + return err + } + return nil +} + +func (t *GenericRecord) UnmarshalCBOR(r io.Reader) (err error) { + *t = GenericRecord{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("GenericRecord: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Type (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Type = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *LegacyBlobSchema) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Cid (string) (string) + if len("cid") > 1000000 { + return xerrors.Errorf("Value in field \"cid\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil { + return err + } + if _, err := cw.WriteString(string("cid")); err != nil { + return err + } + + if len(t.Cid) > 1000000 { + return xerrors.Errorf("Value in field t.Cid was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Cid))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Cid)); err != nil { + return err + } + + // t.MimeType (string) (string) + if len("mimeType") > 1000000 { + return xerrors.Errorf("Value in field \"mimeType\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mimeType"))); err != nil { + return err + } + if _, err := cw.WriteString(string("mimeType")); err != nil { + return err + } + + if len(t.MimeType) > 1000000 { + return xerrors.Errorf("Value in field t.MimeType was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.MimeType))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.MimeType)); err != nil { + return err + } + return nil +} + +func (t *LegacyBlobSchema) UnmarshalCBOR(r io.Reader) (err error) { + *t = LegacyBlobSchema{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("LegacyBlobSchema: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 8) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Cid (string) (string) + case "cid": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Cid = string(sval) + } + // t.MimeType (string) (string) + case "mimeType": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.MimeType = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *BlobSchema) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Ref (atdata.CIDLink) (struct) + if len("ref") > 1000000 { + return xerrors.Errorf("Value in field \"ref\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ref"))); err != nil { + return err + } + if _, err := cw.WriteString(string("ref")); err != nil { + return err + } + + if err := t.Ref.MarshalCBOR(cw); err != nil { + return err + } + + // t.Size (int64) (int64) + if len("size") > 1000000 { + return xerrors.Errorf("Value in field \"size\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("size"))); err != nil { + return err + } + if _, err := cw.WriteString(string("size")); err != nil { + return err + } + + if t.Size >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Size)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Size-1)); err != nil { + return err + } + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blob"))); err != nil { + return err + } + if _, err := cw.WriteString(string("blob")); err != nil { + return err + } + + // t.MimeType (string) (string) + if len("mimeType") > 1000000 { + return xerrors.Errorf("Value in field \"mimeType\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mimeType"))); err != nil { + return err + } + if _, err := cw.WriteString(string("mimeType")); err != nil { + return err + } + + if len(t.MimeType) > 1000000 { + return xerrors.Errorf("Value in field t.MimeType was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.MimeType))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.MimeType)); err != nil { + return err + } + return nil +} + +func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) { + *t = BlobSchema{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("BlobSchema: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 8) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Ref (atdata.CIDLink) (struct) + case "ref": + + { + + if err := t.Ref.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Ref: %w", err) + } + + } + // t.Size (int64) (int64) + case "size": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Size = int64(extraI) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + // t.MimeType (string) (string) + case "mimeType": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.MimeType = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} diff --git a/atproto/atdata/cidlink.go b/atproto/atdata/cidlink.go new file mode 100644 index 000000000..3f129622f --- /dev/null +++ b/atproto/atdata/cidlink.go @@ -0,0 +1,88 @@ +package atdata + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" +) + +// Represents the "cid-link" type from the atproto data model. +// +// Implementation is a simple wrapper around the github.com/ipfs/go-cid "cid.Cid" type. +type CIDLink cid.Cid + +type jsonLink struct { + Link string `json:"$link"` +} + +// Unwraps the inner cid.Cid type (github.com/ipfs/go-cid) +func (ll CIDLink) CID() cid.Cid { + return cid.Cid(ll) +} + +// Returns string representation. +// +// If the CID is "undefined", returns an empty string (note that this is different from how cid.Cid works). +func (ll CIDLink) String() string { + if ll.IsDefined() { + return cid.Cid(ll).String() + } + return "" +} + +// Convenience helper, returns false if CID is "undefined" (golang zero value) +func (ll CIDLink) IsDefined() bool { + return cid.Cid(ll).Defined() +} + +func (ll CIDLink) MarshalJSON() ([]byte, error) { + if !ll.IsDefined() { + return nil, fmt.Errorf("tried to marshal nil or undefined cid-link") + } + jl := jsonLink{ + Link: ll.String(), + } + return json.Marshal(jl) +} + +func (ll *CIDLink) UnmarshalJSON(raw []byte) error { + var jl jsonLink + if err := json.Unmarshal(raw, &jl); err != nil { + return fmt.Errorf("parsing cid-link JSON: %v", err) + } + + c, err := cid.Decode(jl.Link) + if err != nil { + return fmt.Errorf("parsing cid-link CID: %v", err) + } + *ll = CIDLink(c) + return nil +} + +func (ll *CIDLink) MarshalCBOR(w io.Writer) error { + if ll == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if !ll.IsDefined() { + return fmt.Errorf("tried to marshal nil or undefined cid-link") + } + cw := cbg.NewCborWriter(w) + if err := cbg.WriteCid(cw, cid.Cid(*ll)); err != nil { + return fmt.Errorf("failed to write cid-link as CBOR: %w", err) + } + return nil +} + +func (ll *CIDLink) UnmarshalCBOR(r io.Reader) error { + cr := cbg.NewCborReader(r) + c, err := cbg.ReadCid(cr) + if err != nil { + return fmt.Errorf("failed to read cid-link from CBOR: %w", err) + } + *ll = CIDLink(c) + return nil +} diff --git a/atproto/atdata/const.go b/atproto/atdata/const.go new file mode 100644 index 000000000..becd7c980 --- /dev/null +++ b/atproto/atdata/const.go @@ -0,0 +1,30 @@ +package atdata + +const ( + // maximum size of any CBOR data, in any context, in atproto + MAX_CBOR_SIZE = 5 * 1024 * 1024 + // maximum serialized size of an individual atproto record, in CBOR format + MAX_CBOR_RECORD_SIZE = 1 * 1024 * 1024 + // maximum serialized size of an individual atproto record, in JSON format + MAX_JSON_RECORD_SIZE = 2 * 1024 * 1024 + // maximum serialized size of blocks (raw bytes) in an atproto repo stream event (NOT ENFORCED YET) + MAX_STREAM_REPO_DIFF_SIZE = 4 * 1024 * 1024 + // maximum size of a WebSocket frame in atproto event streams (NOT ENFORCED YET) + MAX_STREAM_FRAME_SIZE = MAX_CBOR_SIZE + // maximum size of any individual string inside an atproto record + MAX_RECORD_STRING_LEN = MAX_CBOR_RECORD_SIZE + // maximum size of any individual byte array (bytestring) inside an atproto record + MAX_RECORD_BYTES_LEN = MAX_CBOR_RECORD_SIZE + // limit on size of CID representation (NOT ENFORCED YET) + MAX_CID_BYTES = 100 + // limit on depth of nested containers (objects or arrays) for atproto data (NOT ENFORCED YET) + MAX_CBOR_NESTED_LEVELS = 32 + // maximum number of elements in an object or array in atproto data + MAX_CBOR_CONTAINER_LEN = 128 * 1024 + // largest integer which can be represented in a float64. integers in atproto "should" not be larger than this. (NOT ENFORCED) + MAX_SAFE_INTEGER = 9007199254740991 + // largest negative integer which can be represented in a float64. integers in atproto "should" not go below this. (NOT ENFORCED) + MIN_SAFE_INTEGER = -9007199254740991 + // maximum length of string (UTF-8 bytes) in an atproto object (map) + MAX_OBJECT_KEY_LEN = 8192 +) diff --git a/atproto/atdata/data.go b/atproto/atdata/data.go new file mode 100644 index 000000000..1d824c4b4 --- /dev/null +++ b/atproto/atdata/data.go @@ -0,0 +1,149 @@ +package atdata + +import ( + "encoding/json" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" +) + +// Checks that generic data (object) complies with the atproto data model. +func Validate(obj map[string]any) error { + _, err := parseObject(obj) + return err +} + +// Parses generic data (object) in JSON, validating against the atproto data model at the same time. +// +// The standard library's MarshalJSON can be used to invert this function. +func UnmarshalJSON(b []byte) (map[string]any, error) { + if len(b) > MAX_JSON_RECORD_SIZE { + return nil, fmt.Errorf("exceeded max JSON record size: %d", len(b)) + } + var rawObj map[string]any + err := json.Unmarshal(b, &rawObj) + if err != nil { + return nil, err + } + out, err := parseObject(rawObj) + if err != nil { + return nil, err + } + return out, nil +} + +// Parses generic data (object) in CBOR (specifically, IPLD dag-cbor), validating against the atproto data model at the same time. +func UnmarshalCBOR(b []byte) (map[string]any, error) { + if len(b) > MAX_CBOR_RECORD_SIZE { + return nil, fmt.Errorf("exceeded max CBOR record size: %d", len(b)) + } + var rawObj map[string]any + err := cbor.DecodeInto(b, &rawObj) + if err != nil { + return nil, err + } + out, err := parseObject(rawObj) + if err != nil { + return nil, err + } + return out, nil +} + +// Recursively finds all the "blob" objects from generic atproto data (which has already been parsed). +// +// Returns an array with all Blob instances; does not de-dupe. +func ExtractBlobs(obj map[string]any) []Blob { + return extractBlobsAtom(obj) +} + +func extractBlobsAtom(atom any) []Blob { + out := []Blob{} + switch v := atom.(type) { + case Blob: + out = append(out, v) + case []any: + for _, el := range v { + out = append(out, extractBlobsAtom(el)...) + } + case map[string]any: + for _, val := range v { + out = append(out, extractBlobsAtom(val)...) + } + default: + } + return out +} + +// Serializes generic atproto data (object) to DAG-CBOR bytes +// +// Does not re-validate that data conforms to atproto data model, but does handle Blob, Bytes, and CIDLink as expected. +func MarshalCBOR(obj map[string]any) ([]byte, error) { + return cbor.DumpObject(forCBOR(obj)) +} + +// helper to get generic data in the correct "shape" for serialization with ipfs/go-ipld-cbor +func forCBOR(obj map[string]any) map[string]any { + // NOTE: a faster version might mutate the map in-place instead of copying (many allocations)? + out := make(map[string]any, len(obj)) + for k, val := range obj { + switch v := val.(type) { + case CIDLink: + out[k] = cid.Cid(v) + case Bytes: + out[k] = []byte(v) + case Blob: + out[k] = map[string]interface{}{ + "$type": "blob", + "mimeType": v.MimeType, + "ref": cid.Cid(v.Ref), + "size": v.Size, + } + case syntax.AtIdentifier: + out[k] = v.String() + case *syntax.AtIdentifier: + out[k] = v.String() + case map[string]any: + out[k] = forCBOR(v) + case []any: + out[k] = forCBORArray(v) + default: + out[k] = v + } + } + return out +} + +// recursive helper for forCBOR +func forCBORArray(arr []any) []any { + // NOTE: a faster version might mutate the array in-place instead of copying (many allocations)? + out := make([]any, len(arr)) + for i, val := range arr { + switch v := val.(type) { + case CIDLink: + out[i] = cid.Cid(v) + case Bytes: + out[i] = []byte(v) + case Blob: + out[i] = map[string]interface{}{ + "$type": "blob", + "mimeType": v.MimeType, + "ref": cid.Cid(v.Ref), + "size": v.Size, + } + case syntax.AtIdentifier: + out[i] = v.String() + case *syntax.AtIdentifier: + out[i] = v.String() + case map[string]any: + out[i] = forCBOR(v) + case []any: + out[i] = forCBORArray(v) + default: + out[i] = v + } + } + return out +} diff --git a/atproto/atdata/doc.go b/atproto/atdata/doc.go new file mode 100644 index 000000000..1f78431ea --- /dev/null +++ b/atproto/atdata/doc.go @@ -0,0 +1,18 @@ +/* +Package atdata supports schema-less serializaiton and deserialization of atproto data + +Some restrictions from the data model include: +- string sizes +- array and object element counts +- the "shape" of $bytes and $blob data objects +- $type must contain a non-empty string + +Details are specified at https://atproto.com/specs/data-model + +This package includes types (CIDLink, Bytes, Blob) which are represent the corresponding atproto data model types. These implement JSON and CBOR marshaling in (with whyrusleeping/cbor-gen) the expected way. + +Can parse generic atproto records (or other objects) in JSON or CBOR format in to map[string]interface{}, while validating atproto-specific constraints on data (eg, that cid-link objects have only a single field). + +Has a helper for serializing generic data (map[string]interface{}) to CBOR, which handles converting JSON-style object types (like $link and $bytes) as needed. There is no "MarshalJSON" method; simply use the standard library's `encoding/json`. +*/ +package atdata diff --git a/atproto/atdata/extract.go b/atproto/atdata/extract.go new file mode 100644 index 000000000..b2442a32d --- /dev/null +++ b/atproto/atdata/extract.go @@ -0,0 +1,48 @@ +package atdata + +import ( + "bytes" + "encoding/json" + "fmt" + "io" +) + +// Helper type for extracting record $type from CBOR +type GenericRecord struct { + Type string `json:"$type" cborgen:"$type"` +} + +// Parses the top-level $type field from generic atproto JSON data +func ExtractTypeJSON(b []byte) (string, error) { + var gr GenericRecord + if err := json.Unmarshal(b, &gr); err != nil { + return "", err + } + + return gr.Type, nil +} + +// Parses the top-level $type field from generic atproto CBOR data +func ExtractTypeCBOR(b []byte) (string, error) { + var gr GenericRecord + if err := gr.UnmarshalCBOR(bytes.NewReader(b)); err != nil { + fmt.Printf("bad bytes: %x\n", b) + return "", err + } + + return gr.Type, nil +} + +// Parses top-level $type field from generic atproto CBOR. +// +// Returns that string field, and additional bytes (TODO: the parsed bytes, or remaining bytes?) +func ExtractTypeCBORReader(r io.Reader) (string, []byte, error) { + buf := new(bytes.Buffer) + tr := io.TeeReader(r, buf) + var gr GenericRecord + if err := gr.UnmarshalCBOR(tr); err != nil { + return "", nil, err + } + + return gr.Type, buf.Bytes(), nil +} diff --git a/atproto/atdata/extract_test.go b/atproto/atdata/extract_test.go new file mode 100644 index 000000000..e56f15fa2 --- /dev/null +++ b/atproto/atdata/extract_test.go @@ -0,0 +1,41 @@ +package atdata + +import ( + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtract(t *testing.T) { + assert := assert.New(t) + + // TODO: should this be an error? + tp, err := ExtractTypeJSON([]byte(`{ + "type": "com.example.blah", + "a": 5 + }`)) + assert.NoError(err) + assert.Equal("", tp) + + tp, err = ExtractTypeJSON([]byte(`{ + "$type": "com.example.blah", + "a": 5 + }`)) + assert.NoError(err) + assert.Equal("com.example.blah", tp) + + inFile, err := os.Open("testdata/feedpost_record.cbor") + if err != nil { + t.Fail() + } + cborBytes, err := io.ReadAll(inFile) + if err != nil { + t.Fail() + } + + tp, err = ExtractTypeCBOR(cborBytes) + assert.NoError(err) + assert.Equal("app.bsky.feed.post", tp) +} diff --git a/atproto/atdata/interop_test.go b/atproto/atdata/interop_test.go new file mode 100644 index 000000000..3955014b4 --- /dev/null +++ b/atproto/atdata/interop_test.go @@ -0,0 +1,134 @@ +package atdata + +import ( + "encoding/base64" + "encoding/json" + "io" + "os" + "testing" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/assert" +) + +type DataModelFixture struct { + JSON json.RawMessage `json:"json"` + CBORBase64 string `json:"cbor_base64"` + CID string `json:"cid"` +} + +func TestInteropDataModelFixtures(t *testing.T) { + + f, err := os.Open("testdata/data-model-fixtures.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + fixBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []DataModelFixture + if err := json.Unmarshal(fixBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, row := range fixtures { + testDataModelFixture(t, row) + } +} + +func testDataModelFixture(t *testing.T, row DataModelFixture) { + assert := assert.New(t) + + jsonBytes := []byte(row.JSON) + cborBytes, err := base64.RawStdEncoding.DecodeString(row.CBORBase64) + if err != nil { + t.Fatal(err) + } + + jsonObj, err := UnmarshalJSON(jsonBytes) + assert.NoError(err) + cborObj, err := UnmarshalCBOR(cborBytes) + assert.NoError(err) + + assert.Equal(jsonObj, cborObj) + + cborFromJSON, err := MarshalCBOR(jsonObj) + assert.NoError(err) + cborFromCBOR, err := MarshalCBOR(cborObj) + assert.NoError(err) + + cborObjAgain, err := UnmarshalCBOR(cborFromJSON) + assert.NoError(err) + assert.Equal(jsonObj, cborObjAgain) + + assert.Equal(cborBytes, cborFromJSON) + assert.Equal(cborBytes, cborFromCBOR) + + // 0x71 = dag-cbor, 0x12 = sha2-256, 0 = default length + cidBuilder := cid.V1Builder{Codec: 0x71, MhType: 0x12, MhLength: 0} + cidFromJSON, err := cidBuilder.Sum(cborFromJSON) + assert.NoError(err) + assert.Equal(row.CID, cidFromJSON.String()) + cidFromCBOR, err := cidBuilder.Sum(cborFromCBOR) + assert.NoError(err) + assert.Equal(row.CID, cidFromCBOR.String()) + +} + +type DataModelSimpleFixture struct { + JSON json.RawMessage `json:"json"` +} + +func TestInteropDataModelValid(t *testing.T) { + assert := assert.New(t) + + f, err := os.Open("testdata/data-model-valid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + fixBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []DataModelSimpleFixture + if err := json.Unmarshal(fixBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, row := range fixtures { + _, err := UnmarshalJSON(row.JSON) + assert.NoError(err) + } +} + +func TestInteropDataModelInvalid(t *testing.T) { + assert := assert.New(t) + + f, err := os.Open("testdata/data-model-invalid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + fixBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []DataModelSimpleFixture + if err := json.Unmarshal(fixBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, row := range fixtures { + _, err := UnmarshalJSON(row.JSON) + assert.Error(err) + } +} diff --git a/atproto/atdata/parse.go b/atproto/atdata/parse.go new file mode 100644 index 000000000..23fbd50e8 --- /dev/null +++ b/atproto/atdata/parse.go @@ -0,0 +1,261 @@ +package atdata + +import ( + "encoding" + "encoding/base64" + "fmt" + "reflect" + + "github.com/ipfs/go-cid" +) + +func parseFloat(f float64) (int64, error) { + if f != float64(int64(f)) { + return 0, fmt.Errorf("number is not a safe integer: %f", f) + } + return int64(f), nil +} + +func parseAtom(atom any) (any, error) { + switch v := atom.(type) { + case nil: + return v, nil + case bool: + return v, nil + case *bool: + return *v, nil + case int64: + return v, nil + case *int64: + return *v, nil + case int: + return int64(v), nil + case *int: + return int64(*v), nil + case float64: + return parseFloat(v) + case *float64: + return parseFloat(*v) + case string: + if len(v) > MAX_RECORD_STRING_LEN { + return nil, fmt.Errorf("string too long: %d", len(v)) + } + return v, nil + case *string: + return parseAtom(*v) + case cid.Cid: + return CIDLink(v), nil + case *cid.Cid: + return CIDLink(*v), nil + case []byte: + return Bytes(v), nil + case *[]byte: + return Bytes(*v), nil + case []any: + return parseArray(v) + case *[]any: + return parseArray(*v) + case map[string]any: + return parseMap(v) + case *map[string]any: + return parseMap(*v) + case encoding.TextMarshaler: + s, err := v.MarshalText() + if err != nil { + return nil, fmt.Errorf("failed to marshal text (%s): %w", reflect.TypeOf(v), err) + } + return s, nil + default: + return nil, fmt.Errorf("unexpected type: %s", reflect.TypeOf(v)) + } +} + +func parseArray(l []any) ([]any, error) { + if len(l) > MAX_CBOR_CONTAINER_LEN { + return nil, fmt.Errorf("data array length too long: %d", len(l)) + } + out := make([]any, len(l)) + for i, v := range l { + p, err := parseAtom(v) + if err != nil { + return nil, err + } + out[i] = p + } + return out, nil +} + +func parseMap(obj map[string]any) (any, error) { + if len(obj) > MAX_CBOR_CONTAINER_LEN { + return nil, fmt.Errorf("data object has too many fields: %d", len(obj)) + } + if _, ok := obj["$link"]; ok { + return parseLink(obj) + } + if _, ok := obj["$bytes"]; ok { + return parseBytes(obj) + } + if typeVal, ok := obj["$type"]; ok { + if typeStr, ok := typeVal.(string); ok { + if typeStr == "blob" { + b, err := parseBlob(obj) + if err != nil { + return nil, err + } + return *b, nil + } + if len(typeStr) == 0 { + return nil, fmt.Errorf("$type field must contain a non-empty string") + } + } else { + return nil, fmt.Errorf("$type field must contain a non-empty string") + } + } + // legacy blob type + if len(obj) == 2 { + if _, ok := obj["mimeType"]; ok { + if _, ok := obj["cid"]; ok { + b, err := parseLegacyBlob(obj) + if err != nil { + return nil, err + } + return *b, nil + } + } + } + out := make(map[string]any, len(obj)) + for k, val := range obj { + if len(k) > MAX_OBJECT_KEY_LEN { + return nil, fmt.Errorf("data object key too long: %d", len(k)) + } + atom, err := parseAtom(val) + if err != nil { + return nil, err + } + out[k] = atom + } + return out, nil +} + +func parseLink(obj map[string]any) (CIDLink, error) { + var zero CIDLink + if len(obj) != 1 { + return zero, fmt.Errorf("$link objects must have a single field") + } + v, ok := obj["$link"].(string) + if !ok { + return zero, fmt.Errorf("$link field missing or not a string") + } + c, err := cid.Parse(v) + if err != nil { + return zero, fmt.Errorf("invalid $link CID: %w", err) + } + if !c.Defined() { + return zero, fmt.Errorf("undefined (null) CID in $link") + } + return CIDLink(c), nil +} + +func parseBytes(obj map[string]any) (Bytes, error) { + if len(obj) != 1 { + return nil, fmt.Errorf("$bytes objects must have a single field") + } + v, ok := obj["$bytes"].(string) + if !ok { + return nil, fmt.Errorf("$bytes field missing or not a string") + } + b, err := base64.RawStdEncoding.DecodeString(v) + if err != nil { + return nil, fmt.Errorf("decoding $byte value: %w", err) + } + return Bytes(b), nil +} + +// NOTE: doesn't handle legacy blobs yet! +func parseBlob(obj map[string]any) (*Blob, error) { + if len(obj) != 4 { + return nil, fmt.Errorf("blobs expected to have 4 fields") + } + if obj["$type"] != "blob" { + return nil, fmt.Errorf("blobs expected to have $type=blob") + } + var size int64 + var err error + switch v := obj["size"].(type) { + case int: + size = int64(v) + case int64: + size = v + case float64: + size, err = parseFloat(v) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("blob 'size' missing or not a number") + } + mimeType, ok := obj["mimeType"].(string) + if !ok { + return nil, fmt.Errorf("blob 'mimeType' missing or not a string") + } + rawRef, ok := obj["ref"] + if !ok { + return nil, fmt.Errorf("blob 'ref' missing") + } + var ref CIDLink + switch v := rawRef.(type) { + case map[string]any: + cl, err := parseLink(v) + if err != nil { + return nil, err + } + ref = cl + case cid.Cid: + ref = CIDLink(v) + case CIDLink: + ref = v + default: + return nil, fmt.Errorf("blob 'ref' unexpected type") + } + + return &Blob{ + Size: size, + MimeType: mimeType, + Ref: ref, + }, nil +} + +func parseLegacyBlob(obj map[string]any) (*Blob, error) { + if len(obj) != 2 { + return nil, fmt.Errorf("legacy blobs expected to have 2 fields") + } + var err error + mimeType, ok := obj["mimeType"].(string) + if !ok { + return nil, fmt.Errorf("blob 'mimeType' missing or not a string") + } + cidStr, ok := obj["cid"] + if !ok { + return nil, fmt.Errorf("blob 'cid' missing") + } + c, err := cid.Parse(cidStr) + if err != nil { + return nil, fmt.Errorf("invalid CID: %w", err) + } + return &Blob{ + Size: -1, + MimeType: mimeType, + Ref: CIDLink(c), + }, nil +} + +func parseObject(obj map[string]any) (map[string]any, error) { + out, err := parseMap(obj) + if err != nil { + return nil, err + } + if outObj, ok := out.(map[string]any); ok { + return outObj, nil + } + return nil, fmt.Errorf("top-level datum was not an object") +} diff --git a/atproto/atdata/testdata/data-model-fixtures.json b/atproto/atdata/testdata/data-model-fixtures.json new file mode 100644 index 000000000..f62a12ec8 --- /dev/null +++ b/atproto/atdata/testdata/data-model-fixtures.json @@ -0,0 +1,60 @@ +[ + { + "json": { + "string": "abc", + "unicode": "a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧", + "integer": 123, + "bool": true, + "null": null, + "array": ["abc", "def", "ghi"], + "object": { + "string": "abc", + "number": 123, + "bool": true, + "arr": ["abc", "def", "ghi"] + } + }, + "cbor_base64": "p2Rib29s9WRudWxs9mVhcnJheYNjYWJjY2RlZmNnaGlmb2JqZWN0pGNhcnKDY2FiY2NkZWZjZ2hpZGJvb2z1Zm51bWJlchh7ZnN0cmluZ2NhYmNmc3RyaW5nY2FiY2dpbnRlZ2VyGHtndW5pY29kZXgvYX7DtsOxwqnivZjimI7wk4uT8J+YgPCfkajigI3wn5Gp4oCN8J+Rp+KAjfCfkac", + "cid": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + }, + { + "json": { + "a": { + "$link": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a" + }, + "b": { + "$bytes": "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0" + }, + "c": { + "$type": "blob", + "ref": { + "$link": "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity" + }, + "mimeType": "image/jpeg", + "size": 10000 + } + }, + "cbor_base64": "o2Fh2CpYJQABcRIgZQYqWloA/BbXPGlEI3zLwVscSnI0SJM2iR0JF0GiOdBhYlggnFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI1hY6RjcmVm2CpYJQABVRIgQljP/3j2E2l2l1Y/kmyR5dNXTS6iWuftktbr/COjiJ5kc2l6ZRknEGUkdHlwZWRibG9iaG1pbWVUeXBlamltYWdlL2pwZWc", + "cid": "bafyreihldkhcwijkde7gx4rpkkuw7pl6lbyu5gieunyc7ihactn5bkd2nm" + }, + { + "json": { + "a": { + "b": [ + { + "d": [ + {"$link": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"}, + {"$link": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"} + ], + "e": [ + { "$bytes": "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0" }, + { "$bytes": "iE+sPoHobU9tSIqGI+309LLCcWQIRmEXwxcoDt19tas" } + ] + } + ] + } + }, + "cbor_base64": "oWFhoWFigaJhZILYKlglAAFxEiBlBipaWgD8Ftc8aUQjfMvBWxxKcjRIkzaJHQkXQaI50NgqWCUAAXESIGUGKlpaAPwW1zxpRCN8y8FbHEpyNEiTNokdCRdBojnQYWWCWCCcURGO8suLD2qbjkmuof1BPPILYu7Vdvid7r6wGsLMjVggiE+sPoHobU9tSIqGI+309LLCcWQIRmEXwxcoDt19tas", + "cid": "bafyreid3imdulnhgeytpf6uk7zahjvrsqlofkmm5b5ub2maw4kqus6jp4i" + } +] diff --git a/atproto/atdata/testdata/data-model-invalid.json b/atproto/atdata/testdata/data-model-invalid.json new file mode 100644 index 000000000..03128c54f --- /dev/null +++ b/atproto/atdata/testdata/data-model-invalid.json @@ -0,0 +1,111 @@ +[ + { + "note": "top-level not an object", + "json": "blah" + }, + { + "note": "float", + "json": { + "rcrd": { + "$type": "com.example.blah", + "a": 123.456, + "b": "blah" + } + } + }, + { + "note": "record with $type null", + "json": { + "rcrd": { + "$type": null, + "a": 123, + "b": "blah" + } + } + }, + { + "note": "record with $type wrong type", + "json": { + "rcrd": { + "$type": 123, + "a": 123, + "b": "blah" + } + } + }, + { + "note": "record with empty $type string", + "json": { + "rcrd": { + "$type": "", + "a": 123, + "b": "blah" + } + } + }, + { + "note": "blob with string size", + "json": { + "blb": { + "$type": "blob", + "ref": { + "$link": "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity" + }, + "mimeType": "image/jpeg", + "size": "10000" + } + } + }, + { + "note": "blob with missing key", + "json": { + "blb": { + "$type": "blob", + "mimeType": "image/jpeg", + "size": 10000 + } + } + }, + { + "note": "bytes with wrong field type", + "json": { + "lnk": { + "$bytes": [1,2,3] + } + } + }, + { + "note": "bytes with extra fields", + "json": { + "lnk": { + "$bytes": "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0", + "other": "blah" + } + } + }, + { + "note": "link with wrong field type", + "json": { + "lnk": { + "$link": 1234 + } + } + }, + { + "note": "link with bogus CID", + "json": { + "lnk": { + "$link": "." + } + } + }, + { + "note": "link with extra fields", + "json": { + "lnk": { + "$link": "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity", + "other": "blah" + } + } + } +] diff --git a/atproto/atdata/testdata/data-model-valid.json b/atproto/atdata/testdata/data-model-valid.json new file mode 100644 index 000000000..94f58d271 --- /dev/null +++ b/atproto/atdata/testdata/data-model-valid.json @@ -0,0 +1,48 @@ +[ + { + "note": "trivial record", + "json": { + "rcrd": { + "$type": "com.example.blah", + "a": 123, + "b": "blah" + } + } + }, + { + "note": "float, but integer-like", + "json": { + "rcrd": { + "$type": "com.example.blah", + "a": 123.0, + "b": "blah" + } + } + }, + { + "note": "empty list and object", + "json": { + "rcrd": { + "$type": "com.example.blah", + "a": [], + "b": {} + } + } + }, + { + "note": "list of nullable", + "json": { + "arr": [1,2,null] + } + }, + { + "note": "list of lists", + "json": { + "arr": [ + [1,2,3], + [4,5,6] + ], + "arr2": [null, null, null] + } + } +] diff --git a/atproto/atdata/testdata/feedpost_record.cbor b/atproto/atdata/testdata/feedpost_record.cbor new file mode 100644 index 000000000..e0a61d0fa Binary files /dev/null and b/atproto/atdata/testdata/feedpost_record.cbor differ diff --git a/atproto/auth/http.go b/atproto/auth/http.go new file mode 100644 index 000000000..6b51d2163 --- /dev/null +++ b/atproto/auth/http.go @@ -0,0 +1,97 @@ +package auth + +import ( + "context" + "crypto/subtle" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// HTTP Middleware for atproto admin auth, which is HTTP Basic auth with the username "admin". +// +// This supports multiple admin passwords, which makes it easier to rotate service secrets. +// +// This can be used with `echo.WrapMiddleware` (part of the echo web framework) +func AdminAuthMiddleware(handler http.HandlerFunc, adminPasswords []string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + if ok && username == "admin" { + for _, pw := range adminPasswords { + if subtle.ConstantTimeCompare([]byte(pw), []byte(password)) == 1 { + handler(w, r) + return + } + } + } + w.Header().Set("WWW-Authenticate", `Basic realm="admin", charset="UTF-8"`) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{ + "error": "Unauthorized", + "message": "atproto admin auth required, but missing or incorrect password", + }) + } +} + +// HTTP Middleware for inter-service auth, which is HTTP Bearer with JWT. +// +// 'mandatory' indicates whether valid inter-service auth must be present, or just optional. +func (v *ServiceAuthValidator) Middleware(handler http.HandlerFunc, mandatory bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + + if hdr := r.Header.Get("Authorization"); hdr != "" { + parts := strings.Split(hdr, " ") + if parts[0] != "Bearer" || len(parts) != 2 { + w.Header().Set("WWW-Authenticate", "Bearer") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{ + "error": "Unauthorized", + "message": "atproto service auth required, but missing or incorrect formatting", + }) + return + } + + var lxm *syntax.NSID + uparts := strings.Split(r.URL.Path, "/") + // TODO: should this "fail closed"? eg, reject if not a valid XRPC endpoint + if len(uparts) >= 3 && uparts[1] == "xrpc" { + nsid, err := syntax.ParseNSID(uparts[2]) + if nil == err { + lxm = &nsid + } + } + + did, err := v.Validate(r.Context(), parts[1], lxm) + if err != nil { + w.Header().Set("WWW-Authenticate", "Bearer") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{ + "error": "Unauthorized", + "message": fmt.Sprintf("invalid service auth: %s", err), + }) + return + } + ctx := context.WithValue(r.Context(), "did", did) + handler(w, r.WithContext(ctx)) + return + } + + if mandatory { + w.Header().Set("WWW-Authenticate", "Bearer") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(map[string]string{ + "error": "Unauthorized", + "message": "atproto service auth required", + }) + return + } + handler(w, r) + } +} diff --git a/atproto/auth/http_test.go b/atproto/auth/http_test.go new file mode 100644 index 000000000..85d36de05 --- /dev/null +++ b/atproto/auth/http_test.go @@ -0,0 +1,155 @@ +package auth + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func webHome(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + w.WriteHeader(http.StatusOK) + did, ok := ctx.Value("did").(syntax.DID) + if ok { + w.Write([]byte(did.String())) + } else { + w.Write([]byte("hello world")) + } +} + +func TestAdminAuthMiddleware(t *testing.T) { + assert := assert.New(t) + + pw1 := "secret123" + pw2 := "secret789" + + req := httptest.NewRequest(http.MethodGet, "/", nil) + middle := AdminAuthMiddleware(webHome, []string{pw1, pw2}) + + { + resp := httptest.NewRecorder() + middle(resp, req) + assert.Equal(http.StatusUnauthorized, resp.Code) + } + + { + resp := httptest.NewRecorder() + req.SetBasicAuth("admin", pw1) + middle(resp, req) + assert.Equal(http.StatusOK, resp.Code) + } + + { + resp := httptest.NewRecorder() + req.SetBasicAuth("admin", pw2) + middle(resp, req) + assert.Equal(http.StatusOK, resp.Code) + } + + { + resp := httptest.NewRecorder() + req.SetBasicAuth("wrong", pw2) + middle(resp, req) + assert.Equal(http.StatusUnauthorized, resp.Code) + } + + { + resp := httptest.NewRecorder() + req.SetBasicAuth("admin", "wrong") + middle(resp, req) + assert.Equal(http.StatusUnauthorized, resp.Code) + } +} + +func TestServiceAuthMiddleware(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + iss := syntax.DID("did:example:iss") + aud := "did:example:aud#svc" + lxm := syntax.NSID("com.example.api") + + priv, err := atcrypto.GeneratePrivateKeyP256() + require.NoError(err) + pub, err := priv.PublicKey() + require.NoError(err) + + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{ + DID: iss, + Keys: map[string]identity.VerificationMethod{ + "atproto": { + Type: "Multikey", + PublicKeyMultibase: pub.Multibase(), + }, + }, + }) + + v := ServiceAuthValidator{ + Audience: aud, + Dir: &dir, + } + + { + // optional middleware, no auth + req := httptest.NewRequest(http.MethodGet, "/xrpc/com.example.api", nil) + middle := v.Middleware(webHome, false) + resp := httptest.NewRecorder() + middle(resp, req) + assert.Equal(http.StatusOK, resp.Code) + assert.Equal("hello world", string(resp.Body.Bytes())) + } + + { + // mandatory middleware, no auth + req := httptest.NewRequest(http.MethodGet, "/xrpc/com.example.api", nil) + middle := v.Middleware(webHome, true) + resp := httptest.NewRecorder() + middle(resp, req) + assert.Equal(http.StatusUnauthorized, resp.Code) + } + + { + // mandatory middleware, valid auth + tok, err := SignServiceAuth(iss, aud, time.Minute, &lxm, priv) + require.NoError(err) + req := httptest.NewRequest(http.MethodGet, "/xrpc/com.example.api", nil) + req.Header.Set("Authorization", "Bearer "+tok) + middle := v.Middleware(webHome, true) + resp := httptest.NewRecorder() + middle(resp, req) + assert.Equal(http.StatusOK, resp.Code) + assert.Equal(iss.String(), string(resp.Body.Bytes())) + } + + { + // mangled header + req := httptest.NewRequest(http.MethodGet, "/xrpc/com.example.api", nil) + req.Header.Set("Authorization", "Bearer dummy") + middle := v.Middleware(webHome, false) + resp := httptest.NewRecorder() + middle(resp, req) + assert.Equal(http.StatusUnauthorized, resp.Code) + } + + { + // wrong path + tok, err := SignServiceAuth(iss, aud, time.Minute, &lxm, priv) + require.NoError(err) + req := httptest.NewRequest(http.MethodGet, "/xrpc/com.example.other.api", nil) + req.Header.Set("Authorization", "Bearer "+tok) + middle := v.Middleware(webHome, true) + resp := httptest.NewRecorder() + middle(resp, req) + assert.Equal(http.StatusUnauthorized, resp.Code) + } +} diff --git a/atproto/auth/jwt.go b/atproto/auth/jwt.go new file mode 100644 index 000000000..c2c99e94c --- /dev/null +++ b/atproto/auth/jwt.go @@ -0,0 +1,153 @@ +package auth + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/golang-jwt/jwt/v5" +) + +// TODO: check for uniqueness of JTI (random nonce) to prevent token replay + +type ServiceAuthValidator struct { + // Service DID reference for this validator: a DID with optional #-separated fragment + Audience string + Dir identity.Directory + TimestampLeeway time.Duration +} + +type serviceAuthClaims struct { + jwt.RegisteredClaims + + LexMethod string `json:"lxm,omitempty"` +} + +func (s *ServiceAuthValidator) Validate(ctx context.Context, tokenString string, lexMethod *syntax.NSID) (syntax.DID, error) { + + leeway := s.TimestampLeeway + if leeway == 0 { + leeway = 5 * time.Second + } + + opts := []jwt.ParserOption{ + jwt.WithValidMethods(supportedAlgs), + jwt.WithAudience(s.Audience), + jwt.WithExpirationRequired(), + jwt.WithIssuedAt(), + jwt.WithLeeway(leeway), + } + + token, err := jwt.ParseWithClaims(tokenString, &serviceAuthClaims{}, s.fetchIssuerKeyFunc(ctx), opts...) + if err != nil && errors.Is(err, jwt.ErrTokenSignatureInvalid) { + // if signature validation fails, purge the directory and try again + // TODO: probably need to cache or rate-limit this? + + // do an unvalidated extraction of 'iss' from JWT + insecure := jwt.NewParser(jwt.WithoutClaimsValidation()) + t, _, err := insecure.ParseUnverified(tokenString, &jwt.MapClaims{}) + claims, ok := t.Claims.(*jwt.MapClaims) + if !ok { + return "", jwt.ErrTokenInvalidClaims + } + iss, err := claims.GetIssuer() + if err != nil { + return "", err + } + did, err := syntax.ParseDID(iss) + if err != nil { + return "", fmt.Errorf("%w: invalid DID: %w", jwt.ErrTokenInvalidIssuer, err) + } + + slog.Info("purging directory and retrying service auth signature validation", "did", did) + err = s.Dir.Purge(ctx, did.AtIdentifier()) + if err != nil { + slog.Error("purging identity directory", "did", did, "err", err) + } + token, err = jwt.ParseWithClaims(tokenString, &serviceAuthClaims{}, s.fetchIssuerKeyFunc(ctx), opts...) + } + if err != nil { + return "", err + } + claims, ok := token.Claims.(*serviceAuthClaims) + if !ok { + // TODO: is the error message returned descriptive enough? + return "", jwt.ErrTokenInvalidClaims + } + + if lexMethod != nil && claims.LexMethod != lexMethod.String() { + return "", fmt.Errorf("%w: Lexicon endpoint (LXM)", jwt.ErrTokenInvalidClaims) + } + + // NOTE: KeyFunc has already parsed issuer, so we know it is a valid DID + did := syntax.DID(claims.Issuer) + return did, nil +} + +// resolves public key from identity directory +func (s *ServiceAuthValidator) fetchIssuerKeyFunc(ctx context.Context) func(token *jwt.Token) (any, error) { + return func(token *jwt.Token) (any, error) { + claims, ok := token.Claims.(*serviceAuthClaims) + if !ok { + return nil, jwt.ErrTokenInvalidClaims + } + iss, err := claims.GetIssuer() + if err != nil { + return nil, fmt.Errorf("%w: missing 'iss' claim", jwt.ErrTokenInvalidIssuer) + } + did, err := syntax.ParseDID(iss) + if err != nil { + return nil, fmt.Errorf("%w: invalid DID: %w", jwt.ErrTokenInvalidIssuer, err) + } + // NOTE: this will do handle resolution by default + ident, err := s.Dir.LookupDID(ctx, did) + if err != nil { + return nil, fmt.Errorf("%w: resolving DID (%s): %w", jwt.ErrTokenInvalidIssuer, did, err) + } + return ident.PublicKey() + } +} + +func randomNonce() string { + buf := make([]byte, 16) + rand.Read(buf) + return base64.RawURLEncoding.EncodeToString(buf) +} + +func SignServiceAuth(iss syntax.DID, aud string, ttl time.Duration, lexMethod *syntax.NSID, priv atcrypto.PrivateKey) (string, error) { + claims := serviceAuthClaims{ + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(ttl)), + IssuedAt: jwt.NewNumericDate(time.Now()), + Issuer: iss.String(), + Audience: []string{aud}, + ID: randomNonce(), + }, + } + if lexMethod != nil { + claims.LexMethod = lexMethod.String() + } + + var sm *signingMethodAtproto + + // NOTE: could also have a atcrypto.PrivateKey.Alg() method which returns a string + switch priv.(type) { + case *atcrypto.PrivateKeyP256: + sm = signingMethodES256 + case *atcrypto.PrivateKeyK256: + sm = signingMethodES256K + default: + return "", fmt.Errorf("unknown signing key type: %T", priv) + } + + token := jwt.NewWithClaims(sm, claims) + return token.SignedString(priv) +} diff --git a/atproto/auth/jwt_signing.go b/atproto/auth/jwt_signing.go new file mode 100644 index 000000000..d7007d57b --- /dev/null +++ b/atproto/auth/jwt_signing.go @@ -0,0 +1,88 @@ +package auth + +import ( + "crypto" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/golang-jwt/jwt/v5" +) + +var ( + signingMethodES256K *signingMethodAtproto + signingMethodES256 *signingMethodAtproto + supportedAlgs []string +) + +// Implementation of jwt.SigningMethod for the `atproto/atcrypto` types. +type signingMethodAtproto struct { + alg string + hash crypto.Hash + toOutSig toOutSig + sigLen int +} + +type toOutSig func(sig []byte) []byte + +func init() { + // tells JWT library to serialize 'aud' as regular string, not array of strings (when signing) + jwt.MarshalSingleStringAsArray = false + + signingMethodES256K = &signingMethodAtproto{ + alg: "ES256K", + hash: crypto.SHA256, + toOutSig: toES256K, + sigLen: 64, + } + jwt.RegisterSigningMethod(signingMethodES256K.Alg(), func() jwt.SigningMethod { + return signingMethodES256K + }) + signingMethodES256 = &signingMethodAtproto{ + alg: "ES256", + hash: crypto.SHA256, + toOutSig: toES256, + sigLen: 64, + } + jwt.RegisterSigningMethod(signingMethodES256.Alg(), func() jwt.SigningMethod { + return signingMethodES256 + }) + supportedAlgs = []string{signingMethodES256K.Alg(), signingMethodES256.Alg()} +} + +func (sm *signingMethodAtproto) Verify(signingString string, sig []byte, key interface{}) error { + pub, ok := key.(atcrypto.PublicKey) + if !ok { + return jwt.ErrInvalidKeyType + } + + if !sm.hash.Available() { + return jwt.ErrHashUnavailable + } + + if len(sig) != sm.sigLen { + return jwt.ErrTokenSignatureInvalid + } + + // NOTE: important to use using "lenient" variant here + return pub.HashAndVerifyLenient([]byte(signingString), sig) +} + +func (sm *signingMethodAtproto) Sign(signingString string, key interface{}) ([]byte, error) { + priv, ok := key.(atcrypto.PrivateKey) + if !ok { + return nil, jwt.ErrInvalidKeyType + } + + return priv.HashAndSign([]byte(signingString)) +} + +func (sm *signingMethodAtproto) Alg() string { + return sm.alg +} + +func toES256K(sig []byte) []byte { + return sig[:64] +} + +func toES256(sig []byte) []byte { + return sig[:64] +} diff --git a/atproto/auth/jwt_test.go b/atproto/auth/jwt_test.go new file mode 100644 index 000000000..da4ec7f26 --- /dev/null +++ b/atproto/auth/jwt_test.go @@ -0,0 +1,156 @@ +package auth + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/golang-jwt/jwt/v5" + "github.com/stretchr/testify/assert" +) + +// Returns an early-2024 timestamp as a point in time for validating known JWTs (which contain expires-at) +func testTime() time.Time { + return time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +} + +func validateMinimal(token string, iss, aud string, pub atcrypto.PublicKey) error { + + p := jwt.NewParser( + jwt.WithValidMethods(supportedAlgs), + jwt.WithTimeFunc(testTime), + jwt.WithIssuer(iss), + jwt.WithAudience(aud), + ) + _, err := p.Parse(token, func(tok *jwt.Token) (any, error) { + return pub, nil + }) + if err != nil { + return fmt.Errorf("failed to parse auth header JWT: %w", err) + } + return nil +} + +func TestSignatureMethods(t *testing.T) { + assert := assert.New(t) + + jwtTestFixtures := []struct { + name string + pubkey string + iss string + aud string + jwt string + }{ + { + name: "secp256k1 (K-256)", + pubkey: "did:key:zQ3shscXNYZQZSPwegiv7uQZZV5kzATLBRtgJhs7uRY7pfSk4", + iss: "did:example:iss", + aud: "did:example:aud", + jwt: "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NksifQ.eyJpc3MiOiJkaWQ6ZXhhbXBsZTppc3MiLCJhdWQiOiJkaWQ6ZXhhbXBsZTphdWQiLCJleHAiOjE3MTM1NzEwMTJ9.J_In_PQCMjygeeoIKyjybORD89ZnEy1bZTd--sdq_78qv3KCO9181ZAh-2Pl0qlXZjfUlxgIa6wiak2NtsT98g", + }, + { + name: "secp256k1 (K-256)", + pubkey: "did:key:zQ3shqKrpHzQ5HDfhgcYMWaFcpBK3SS39wZLdTjA5GeakX8G5", + iss: "did:example:iss", + aud: "did:example:aud", + jwt: "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NksifQ.eyJhdWQiOiJkaWQ6ZXhhbXBsZTphdWQiLCJpc3MiOiJkaWQ6ZXhhbXBsZTppc3MiLCJleHAiOjE3MTM1NzExMzJ9.itNeYcF5oFMZIGxtnbJhE4McSniv_aR-Yk1Wj8uWk1K8YjlS2fzuJMo0-fILV3payETxn6r45f0FfpTaqY0EZQ", + }, + { + name: "P-256", + pubkey: "did:key:zDnaeXRDKRCEUoYxi8ZJS2pDsgfxUh3pZiu3SES9nbY4DoART", + iss: "did:example:iss", + aud: "did:example:aud", + jwt: "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJpc3MiOiJkaWQ6ZXhhbXBsZTppc3MiLCJhdWQiOiJkaWQ6ZXhhbXBsZTphdWQiLCJleHAiOjE3MTM1NzE1NTR9.FFRLm7SGbDUp6cL0WoCs0L5oqNkjCXB963TqbgI-KxIjbiqMQATVCalcMJx17JGTjMmfVHJP6Op_V4Z0TTjqog", + }, + } + + for _, fix := range jwtTestFixtures { + + pubk, err := atcrypto.ParsePublicDIDKey(fix.pubkey) + if err != nil { + t.Fatal(err) + } + + assert.NoError(validateMinimal(fix.jwt, fix.iss, fix.aud, pubk)) + } +} + +func testSigningValidation(t *testing.T, priv atcrypto.PrivateKey) { + assert := assert.New(t) + ctx := context.Background() + + iss := syntax.DID("did:example:iss") + aud := "did:example:aud#svc" + lxm := syntax.NSID("com.example.api") + + priv, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + t.Fatal(err) + } + pub, err := priv.PublicKey() + if err != nil { + t.Fatal(err) + } + + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{ + DID: iss, + Keys: map[string]identity.VerificationMethod{ + "atproto": { + Type: "Multikey", + PublicKeyMultibase: pub.Multibase(), + }, + }, + }) + + v := ServiceAuthValidator{ + Audience: aud, + Dir: &dir, + } + + t1, err := SignServiceAuth(iss, aud, time.Minute, nil, priv) + if err != nil { + t.Fatal(err) + } + d1, err := v.Validate(ctx, t1, nil) + assert.NoError(err) + assert.Equal(d1, iss) + _, err = v.Validate(ctx, t1, &lxm) + assert.Error(err) + + t2, err := SignServiceAuth(iss, aud, time.Minute, &lxm, priv) + if err != nil { + t.Fatal(err) + } + d2, err := v.Validate(ctx, t2, nil) + assert.NoError(err) + assert.Equal(d2, iss) + _, err = v.Validate(ctx, t2, &lxm) + assert.NoError(err) + + _, err = v.Validate(ctx, t2, nil) + assert.NoError(err) + _, err = v.Validate(ctx, t2, &lxm) + assert.NoError(err) +} + +func TestP256SigningValidation(t *testing.T) { + priv, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + t.Fatal(err) + } + testSigningValidation(t, priv) +} + +func TestK256SigningValidation(t *testing.T) { + priv, err := atcrypto.GeneratePrivateKeyK256() + if err != nil { + t.Fatal(err) + } + testSigningValidation(t, priv) +} diff --git a/atproto/auth/oauth/HACKING.md b/atproto/auth/oauth/HACKING.md new file mode 100644 index 000000000..79a4f7400 --- /dev/null +++ b/atproto/auth/oauth/HACKING.md @@ -0,0 +1,26 @@ + +## Package Structure + +`oauth.ClientApp` +- represents an overall application or service; helps establish and manage oauth.ClientSession +- wraps and manages client metadata, client attestation secret (for confidential clients), request and session storage + +`oauth.ClientSession` +- represents an established user session, wrapping DPoP key, tokens, and other metadata +- implements client.AuthMethod, for use with ApiClient +- automates token refresh; for confidential clients requires ref to client secret +- triggers callback when session data are updated (nonce, tokens) + +`oauth.ClientAuthStore` +- interface for persistent storage systems for auth request and session metadata, including secrets and DPoP private keys + +`oauth.Resolver` +- currently always resolves direct from the network; may add flexible caching or interface abstraction in the future + + +## Implementation Details + +- starts DPoP at PAR (specification is flexible about this) +- requires ES256 (P-256) for DPoP and client attestation private keys; though flexible interface types are used in the API +- scopes are configured as part of client metadata, and the same for each session + diff --git a/atproto/auth/oauth/cmd/oauth-web-demo/README.md b/atproto/auth/oauth/cmd/oauth-web-demo/README.md new file mode 100644 index 000000000..358fc8ef6 --- /dev/null +++ b/atproto/auth/oauth/cmd/oauth-web-demo/README.md @@ -0,0 +1,21 @@ + +OAuth SDK Web App Example +========================= + +This is a minimal Go web app showing how to use the OAuth client SDK. + +To get started, generated a `.env` file with the following variables: + +- `SESSION_SECRET` (required) is a random string for secure cookies, you can generate one with `openssl rand -hex 16` +- `CLIENT_HOSTNAME` (optional) is a public web hostname at which the running web app can be reached on the public web, with `https://`. It needs to actually be reachable by remote servers, not just your local web browser; you can use a service like `ngrok` if experimenting on a laptop. Or, if you leave this blank, the app will run as a "localhost dev app". +- `CLIENT_SECRET_KEY` (optional) is used to run as a "confidential" client, with client attestation. You can generate a private key with the `goat` CLI tool (`goat key generate -t P-256`) + +And example file might look like: + +``` +SESSION_SECRET=49922828917dc6ac2f2fd2cca78735c3 +CLIENT_SECRET_KEY=z42twLj2gZeJSeRgZ4yPyEb6Yg6nawhU2W8y2ETDDFFyvwym +CLIENT_HOSTNAME=a9a7c2e14c.ngrok-free.app +``` + +Then run the demo (`go run .`) and connect with a web browser. diff --git a/atproto/auth/oauth/cmd/oauth-web-demo/base.html b/atproto/auth/oauth/cmd/oauth-web-demo/base.html new file mode 100644 index 000000000..78d3daeb6 --- /dev/null +++ b/atproto/auth/oauth/cmd/oauth-web-demo/base.html @@ -0,0 +1,36 @@ + + + + + + + + + atproto OAuth demo (indigo) + + +
+
+

atproto OAuth demo (indigo)

+
+ +
+
+
+ {{ template "content" . }} +
+
+ + diff --git a/atproto/auth/oauth/cmd/oauth-web-demo/home.html b/atproto/auth/oauth/cmd/oauth-web-demo/home.html new file mode 100644 index 000000000..0ac4755f7 --- /dev/null +++ b/atproto/auth/oauth/cmd/oauth-web-demo/home.html @@ -0,0 +1,5 @@ +{{ define "content" }} +

This is a minimal web app showing how to use the indigo OAuth client SDK to authenticate users. You can read more in the atproto OAuth Specification + +

Click "Login" above to get started. +{{ end }} diff --git a/atproto/auth/oauth/cmd/oauth-web-demo/login.html b/atproto/auth/oauth/cmd/oauth-web-demo/login.html new file mode 100644 index 000000000..1c050e580 --- /dev/null +++ b/atproto/auth/oauth/cmd/oauth-web-demo/login.html @@ -0,0 +1,13 @@ +{{ define "content" }} +

+

Login with atproto

+
+

Provide your handle or DID to authorize an existing account with PDS. +
You can also supply a PDS/entryway URL (eg, https://pds.example.com).

+
+ + +
+
+
+{{ end }} diff --git a/atproto/auth/oauth/cmd/oauth-web-demo/main.go b/atproto/auth/oauth/cmd/oauth-web-demo/main.go new file mode 100644 index 000000000..c8623c30b --- /dev/null +++ b/atproto/auth/oauth/cmd/oauth-web-demo/main.go @@ -0,0 +1,369 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "html/template" + "log/slog" + "net/http" + "os" + "slices" + + _ "embed" + _ "github.com/joho/godotenv/autoload" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/auth/oauth" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/gorilla/sessions" + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "oauth-web-demo", + Usage: "atproto OAuth web server demo", + Action: runServer, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "session-secret", + Usage: "random string/token used for session cookie security", + Required: true, + Sources: cli.EnvVars("SESSION_SECRET"), + }, + &cli.StringFlag{ + Name: "hostname", + Usage: "public host name for this client (if not localhost dev mode)", + Sources: cli.EnvVars("CLIENT_HOSTNAME"), + }, + &cli.StringFlag{ + Name: "client-secret-key", + Usage: "confidential client secret key. should be P-256 private key in multibase encoding", + Sources: cli.EnvVars("CLIENT_SECRET_KEY"), + }, + &cli.StringFlag{ + Name: "client-secret-key-id", + Usage: "key id for client-secret-key", + Value: "primary", + Sources: cli.EnvVars("CLIENT_SECRET_KEY_ID"), + }, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + if err := app.Run(context.Background(), os.Args); err != nil { + slog.Error("command failed", "error", err) + os.Exit(-1) + } +} + +type Server struct { + CookieStore *sessions.CookieStore + Dir identity.Directory + OAuth *oauth.ClientApp +} + +//go:embed "base.html" +var tmplBaseText string + +//go:embed "home.html" +var tmplHomeText string +var tmplHome = template.Must(template.Must(template.New("home.html").Parse(tmplBaseText)).Parse(tmplHomeText)) + +//go:embed "login.html" +var tmplLoginText string +var tmplLogin = template.Must(template.Must(template.New("login.html").Parse(tmplBaseText)).Parse(tmplLoginText)) + +//go:embed "post.html" +var tmplPostText string +var tmplPost = template.Must(template.Must(template.New("post.html").Parse(tmplBaseText)).Parse(tmplPostText)) + +func runServer(ctx context.Context, cmd *cli.Command) error { + + // the 'account:email' scope is requested only as a demo of users not granting a permission during auth flow + scopes := []string{"atproto", "repo:app.bsky.feed.post?action=create", "account:email"} + bind := ":8080" + + var config oauth.ClientConfig + hostname := cmd.String("hostname") + if hostname == "" { + config = oauth.NewLocalhostConfig( + fmt.Sprintf("http://127.0.0.1%s/oauth/callback", bind), + scopes, + ) + slog.Info("configuring localhost OAuth client", "CallbackURL", config.CallbackURL) + } else { + config = oauth.NewPublicConfig( + fmt.Sprintf("https://%s/oauth/client-metadata.json", hostname), + fmt.Sprintf("https://%s/oauth/callback", hostname), + scopes, + ) + } + + // If a client secret key is provided (as a multibase string), turn this in to a confidential client + if cmd.String("client-secret-key") != "" && hostname != "" { + priv, err := atcrypto.ParsePrivateMultibase(cmd.String("client-secret-key")) + if err != nil { + return err + } + if err := config.SetClientSecret(priv, cmd.String("client-secret-key-id")); err != nil { + return err + } + slog.Info("configuring confidential OAuth client") + } + + oauthClient := oauth.NewClientApp(&config, oauth.NewMemStore()) + + srv := Server{ + CookieStore: sessions.NewCookieStore([]byte(cmd.String("session-secret"))), + Dir: identity.DefaultDirectory(), + OAuth: oauthClient, + } + + http.HandleFunc("GET /", srv.Homepage) + http.HandleFunc("GET /oauth/client-metadata.json", srv.ClientMetadata) + http.HandleFunc("GET /oauth/jwks.json", srv.JWKS) + http.HandleFunc("GET /oauth/login", srv.OAuthLogin) + http.HandleFunc("POST /oauth/login", srv.OAuthLogin) + http.HandleFunc("GET /oauth/callback", srv.OAuthCallback) + http.HandleFunc("GET /oauth/refresh", srv.OAuthRefresh) + http.HandleFunc("GET /oauth/logout", srv.OAuthLogout) + http.HandleFunc("GET /bsky/post", srv.Post) + http.HandleFunc("POST /bsky/post", srv.Post) + + slog.Info("starting http server", "bind", bind) + if err := http.ListenAndServe(bind, nil); err != nil { + slog.Error("http shutdown", "err", err) + } + return nil +} + +func (s *Server) currentSessionDID(r *http.Request) (*syntax.DID, string) { + sess, _ := s.CookieStore.Get(r, "oauth-demo") + accountDID, ok := sess.Values["account_did"].(string) + if !ok || accountDID == "" { + return nil, "" + } + did, err := syntax.ParseDID(accountDID) + if err != nil { + return nil, "" + } + sessionID, ok := sess.Values["session_id"].(string) + if !ok || sessionID == "" { + return nil, "" + } + + return &did, sessionID +} + +func strPtr(raw string) *string { + return &raw +} + +func (s *Server) ClientMetadata(w http.ResponseWriter, r *http.Request) { + slog.Info("client metadata request", "url", r.URL, "host", r.Host) + + meta := s.OAuth.Config.ClientMetadata() + if s.OAuth.Config.IsConfidential() { + meta.JWKSURI = strPtr(fmt.Sprintf("https://%s/oauth/jwks.json", r.Host)) + } + meta.ClientName = strPtr("indigo atp-oauth-demo") + meta.ClientURI = strPtr(fmt.Sprintf("https://%s", r.Host)) + + // internal consistency check + if err := meta.Validate(s.OAuth.Config.ClientID); err != nil { + slog.Error("validating client metadata", "err", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(meta); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +func (s *Server) JWKS(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + body := s.OAuth.Config.PublicJWKS() + if err := json.NewEncoder(w).Encode(body); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} + +func (s *Server) Homepage(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // attempts to load Session to display links + did, sessionID := s.currentSessionDID(r) + if did == nil { + tmplHome.Execute(w, nil) + return + } + + _, err := s.OAuth.ResumeSession(ctx, *did, sessionID) + if err != nil { + tmplHome.Execute(w, nil) + return + } + tmplHome.Execute(w, did) +} + +func (s *Server) OAuthLogin(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + if r.Method != "POST" { + tmplLogin.Execute(w, nil) + return + } + + if err := r.ParseForm(); err != nil { + http.Error(w, fmt.Errorf("parsing form data: %w", err).Error(), http.StatusBadRequest) + return + } + + username := r.PostFormValue("username") + + slog.Info("OAuthLogin", "client_id", s.OAuth.Config.ClientID, "callback_url", s.OAuth.Config.CallbackURL) + + redirectURL, err := s.OAuth.StartAuthFlow(ctx, username) + if err != nil { + http.Error(w, fmt.Errorf("OAuth login failed: %w", err).Error(), http.StatusBadRequest) + return + } + + http.Redirect(w, r, redirectURL, http.StatusFound) + return +} + +func (s *Server) OAuthCallback(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + params := r.URL.Query() + slog.Info("received callback", "params", params) + + sessData, err := s.OAuth.ProcessCallback(ctx, r.URL.Query()) + if err != nil { + http.Error(w, fmt.Errorf("processing OAuth callback: %w", err).Error(), http.StatusBadRequest) + return + } + + if !slices.Equal(sessData.Scopes, s.OAuth.Config.Scopes) { + slog.Warn("session auth scopes did not match those requested", "requested", s.OAuth.Config.Scopes, "granted", sessData.Scopes) + } + + // create signed cookie session, indicating account DID + sess, _ := s.CookieStore.Get(r, "oauth-demo") + sess.Values["account_did"] = sessData.AccountDID.String() + sess.Values["session_id"] = sessData.SessionID + if err := sess.Save(r, w); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + slog.Info("login successful", "did", sessData.AccountDID.String()) + http.Redirect(w, r, "/bsky/post", http.StatusFound) +} + +func (s *Server) OAuthRefresh(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + did, sessionID := s.currentSessionDID(r) + if did == nil { + // TODO: supposed to set a WWW header; and could redirect? + http.Error(w, "not authenticated", http.StatusUnauthorized) + return + } + + oauthSess, err := s.OAuth.ResumeSession(ctx, *did, sessionID) + if err != nil { + http.Error(w, "not authenticated", http.StatusUnauthorized) + return + } + + _, err = oauthSess.RefreshTokens(ctx) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + s.OAuth.Store.SaveSession(ctx, *oauthSess.Data) + slog.Info("refreshed tokens") + http.Redirect(w, r, "/", http.StatusFound) +} + +func (s *Server) OAuthLogout(w http.ResponseWriter, r *http.Request) { + + // revoke tokens and delete session from auth store + did, sessionID := s.currentSessionDID(r) + if did != nil { + if err := s.OAuth.Logout(r.Context(), *did, sessionID); err != nil { + slog.Error("failed to delete session", "did", did, "err", err) + } + } + + // wipe all secure cookie session data + sess, _ := s.CookieStore.Get(r, "oauth-demo") + sess.Values = make(map[any]any) + err := sess.Save(r, w) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + slog.Info("logged out") + http.Redirect(w, r, "/", http.StatusFound) +} + +func (s *Server) Post(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + slog.Info("in post handler") + + did, sessionID := s.currentSessionDID(r) + if did == nil { + // TODO: supposed to set a WWW header; and could redirect? + http.Error(w, "not authenticated", http.StatusUnauthorized) + return + } + + if r.Method != "POST" { + tmplPost.Execute(w, did) + return + } + + oauthSess, err := s.OAuth.ResumeSession(ctx, *did, sessionID) + if err != nil { + http.Error(w, "not authenticated", http.StatusUnauthorized) + return + } + c := oauthSess.APIClient() + + if err := r.ParseForm(); err != nil { + http.Error(w, fmt.Errorf("parsing form data: %w", err).Error(), http.StatusBadRequest) + return + } + text := r.PostFormValue("post_text") + + body := map[string]any{ + "repo": c.AccountDID.String(), + "collection": "app.bsky.feed.post", + "record": map[string]any{ + "$type": "app.bsky.feed.post", + "text": text, + "createdAt": syntax.DatetimeNow(), + }, + } + + slog.Info("attempting post...", "text", text) + if err := c.Post(ctx, "com.atproto.repo.createRecord", body, nil); err != nil { + http.Error(w, fmt.Errorf("posting failed: %w", err).Error(), http.StatusBadRequest) + return + } + + http.Redirect(w, r, "/bsky/post", http.StatusFound) +} diff --git a/atproto/auth/oauth/cmd/oauth-web-demo/post.html b/atproto/auth/oauth/cmd/oauth-web-demo/post.html new file mode 100644 index 000000000..27c15a05a --- /dev/null +++ b/atproto/auth/oauth/cmd/oauth-web-demo/post.html @@ -0,0 +1,6 @@ +{{ define "content" }} +
+ + +
+{{ end }} diff --git a/atproto/auth/oauth/doc.go b/atproto/auth/oauth/doc.go new file mode 100644 index 000000000..ec7f5887d --- /dev/null +++ b/atproto/auth/oauth/doc.go @@ -0,0 +1,145 @@ +/* +OAuth implementation for atproto, currently focused on clients. + +Feature set includes: + + - client and server metadata resolution + - PKCE: computing and verifying challenges + - DPoP client implementation: JWT signing and nonces for requests to Auth Server and Resource Server + - PAR client submission + - both public and confidential clients, with support for signed client attestations in the later case + +Most OAuth client applications will use the high-level [ClientApp] and supporting interfaces to manage session logins, persistence, and token refreshes. Lower-level components are designed to be used in isolation if needed. + +This package does not contain supporting code for atproto permissions or permission sets. It treats scopes as simple strings. + +# Quickstart + +Create a single [ClientApp] instance during service setup that will be used (concurrently) across all users and sessions: + + config := oauth.NewPublicConfig( + "https://app.example.com/client-metadata.json", + "https://app.example.com/oauth/callback", + []string{"atproto", "repo:app.bsky.feed.post?action=create"}, + ) + + // clients are "public" by default, but if they have secure access to a secret attestation key can be "confidential" + if CLIENT_SECRET_KEY != "" { + priv, err := crypto.ParsePrivateMultibase(CLIENT_SECRET_KEY) + if err != nil { + return err + } + if err := config.SetClientSecret(priv, "example1"); err != nil { + return err + } + } + + oauthApp := oauth.NewClientApp(&config, oauth.NewMemStore()) + +For a real service, you would want to use a database or other persistant implementation of the [ClientAuthStore] interface instead of [MemStore]. Otherwise all user sessions are dropped every time the process restarts. + +The client metadata document needs to be served at the URL indicated by the 'client_id'. This can be done statically, or dynamically generated and served from the configuration: + + http.HandleFunc("GET /client-metadata.json", HandleClientMetadata) + + func HandleClientMetadata(w http.ResponseWriter, r *http.Request) { + doc := oauthApp.Config.ClientMetadata() + + // if this is is a confidential client, need to set doc.JWKSURI, and implement a handler + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(doc); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + +The login auth flow starts with a user identifier, which could be an atproto handle, DID, or an auth server URL (eg, a PDS). The high-level [ClientApp.StartAuthFlow] method will resolve the identifier, send an auth request (PAR) to the server, persist request metadata in the [ClientAuthStore], and return a redirect URL for the user to visit (usually the PDS): + + http.HandleFunc("GET /oauth/login", HandleLogin) + + func HandleLogin(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // parse login identifier from the request + identifier := "..." + + redirectURL, err := oauthApp.StartAuthFlow(ctx, identifier) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + http.Redirect(w, r, redirectURL, http.StatusFound) + } + +The service then waits for a callback request on the configured endpoint. The [ClientApp.ProcessCallback] method will load the earlier request metadata from the [ClientAuthStore], send an initial token request to the auth server, and validate that the session is consistent with the identifier from the beginning of the login flow. + + http.HandleFunc("GET /oauth/callback", HandleOAuthCallback) + + func HandleOAuthCallback(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + sessData, err := oauthApp.ProcessCallback(ctx, r.URL.Query()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + // web services might record the DID and session ID in a secure session cookie + _ = sessData.AccountDID + _ = sessData.SessionID + + // the returned scopes might not include all of those requested + _ = sessData.Scopes + + http.Redirect(w, r, "/app", http.StatusFound) + } + +Sessions can be resumed and used to make authenticated API calls to the user's host: + + // web services might use a secure session cookie to determine user's DID for a request + did := syntax.DID("did:plc:abc123") + sessionID := "xyz" + + sess, err := oauthApp.ResumeSession(ctx, did, sessionID) + if err != nil { + return err + } + + c := sess.APIClient() + + body := map[string]any{ + "repo": *c.AccountDID, + "collection": "app.bsky.feed.post", + "record": map[string]any{ + "$type": "app.bsky.feed.post", + "text": "Hello World via OAuth!", + "createdAt": syntax.DatetimeNow(), + }, + } + + if err := c.Post(ctx, "com.atproto.repo.createRecord", body, nil); err != nil { + return err + } + +The [ClientSession] will handle nonce updates and token refreshes, and persist the results in the [ClientAuthStore]. + +To log out a user, use the [ClientApp.Logout] helper method, which revokes their tokens (if supported by the AS) and deletes their session from the [ClientAuthStore]: + + if err := oauthApp.Logout(r.Context(), did, sessionID); err != nil { + return err + } + +# Authorization-only Situations + +Some applications might only use atproto OAuth for authorization (authn). For example, "Login with Atmospehre", where the application does not need to access additional account metadata (such as account email), or access any restricted account resources (eg, write to atproto repository). + +In this scenario, the client app still needs to do an initial token request, to confirm the account identifier. But the returned session tokens will never be used, and do not need to be persisted. + +In these scenarios, applications could use an implementation of [ClientAuthStore] which does not actually persist the session data when [ClientAuthStore.SaveSession] is called. Or, the application could immediately call [ClientAuthStore.DeleteSession] after [ClientApp.ProcessCallback] returns. + +# Multiple Sessions Per Account + +In the traditional web app backend scenario, a single account (DID) might have multiple active sessions. For example, a user might log in from a browser on their laptop and on a mobile device at the same time. The user must go through the entire flow on each device (or browser) to authenticate the user. To prevent a new session from "clobbering" existing sessions (including tokens), this package supports multiple concurrent sessions per account, distinguished by a session ID. The random 'state' token from the auth flow is re-used by default. + +In other scenarios, multiple sessions are not needed or desirable. For example, an integration backend, or tool with very short session lifetimes. In these scenarios, implementations of the [ClientAuthStore] interface could ignore the session ID. Or the [ClientApp] could be configured with an ephemeral [ClientAuthStore] (to support auth flows), and managed the session data returned by [ClientApp.ProcessCallback] using separate session storage logic. +*/ +package oauth diff --git a/atproto/auth/oauth/jwt_signing.go b/atproto/auth/oauth/jwt_signing.go new file mode 100644 index 000000000..4981189d4 --- /dev/null +++ b/atproto/auth/oauth/jwt_signing.go @@ -0,0 +1,87 @@ +package oauth + +import ( + "crypto" + "fmt" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/golang-jwt/jwt/v5" +) + +// NOTE: this file is copied from indigo:atproto/auth/jwt_signing.go, with the K-256 (ES256) support removed + +var ( + signingMethodES256 *signingMethodAtproto + supportedAlgs []string +) + +// Implementation of jwt.SigningMethod for the `atproto/atcrypto` types. +type signingMethodAtproto struct { + alg string + hash crypto.Hash + toOutSig toOutSig + sigLen int +} + +type toOutSig func(sig []byte) []byte + +func init() { + // tells JWT library to serialize 'aud' as regular string, not array of strings (when signing) + jwt.MarshalSingleStringAsArray = false + + signingMethodES256 = &signingMethodAtproto{ + alg: "ES256", + hash: crypto.SHA256, + toOutSig: toES256, + sigLen: 64, + } + jwt.RegisterSigningMethod(signingMethodES256.Alg(), func() jwt.SigningMethod { + return signingMethodES256 + }) + supportedAlgs = []string{signingMethodES256.Alg()} +} + +func (sm *signingMethodAtproto) Verify(signingString string, sig []byte, key interface{}) error { + pub, ok := key.(atcrypto.PublicKey) + if !ok { + return jwt.ErrInvalidKeyType + } + + if !sm.hash.Available() { + return jwt.ErrHashUnavailable + } + + if len(sig) != sm.sigLen { + return jwt.ErrTokenSignatureInvalid + } + + // NOTE: important to use using "lenient" variant here. atproto cryptography is strict about details like low-S elliptic curve signatures, but OAuth cryptography is not, and we want to be interoperable with general purpose OAuth implementations + return pub.HashAndVerifyLenient([]byte(signingString), sig) +} + +func (sm *signingMethodAtproto) Sign(signingString string, key interface{}) ([]byte, error) { + priv, ok := key.(atcrypto.PrivateKey) + if !ok { + return nil, jwt.ErrInvalidKeyType + } + + return priv.HashAndSign([]byte(signingString)) +} + +func (sm *signingMethodAtproto) Alg() string { + return sm.alg +} + +func toES256(sig []byte) []byte { + return sig[:64] +} + +func keySigningMethod(key atcrypto.PrivateKey) (jwt.SigningMethod, error) { + switch key.(type) { + case *atcrypto.PrivateKeyP256: + return signingMethodES256, nil + case *atcrypto.PrivateKeyK256: + return nil, fmt.Errorf("only P-256 (ES256) private keys supported for atproto OAuth") + } + return nil, fmt.Errorf("unknown key type: %T", key) +} diff --git a/atproto/auth/oauth/memstore.go b/atproto/auth/oauth/memstore.go new file mode 100644 index 000000000..5cdf12130 --- /dev/null +++ b/atproto/auth/oauth/memstore.go @@ -0,0 +1,92 @@ +package oauth + +import ( + "context" + "fmt" + "sync" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Simple in-memory implementation of [ClientAuthStore], for use in development and demos. +// +// This is not appropriate even casual real-world use: all users will be logged-out every time the process is restarted. +type MemStore struct { + requests map[string]AuthRequestData + sessions map[string]ClientSessionData + + lk sync.Mutex +} + +var _ ClientAuthStore = &MemStore{} + +func NewMemStore() *MemStore { + return &MemStore{ + requests: make(map[string]AuthRequestData), + sessions: make(map[string]ClientSessionData), + } +} + +func memKey(did syntax.DID, sessionID string) string { + return fmt.Sprintf("%s/%s", did, sessionID) +} + +func (m *MemStore) GetSession(ctx context.Context, did syntax.DID, sessionID string) (*ClientSessionData, error) { + m.lk.Lock() + defer m.lk.Unlock() + + sess, ok := m.sessions[memKey(did, sessionID)] + if !ok { + return nil, fmt.Errorf("session not found: %s", did) + } + return &sess, nil +} + +func (m *MemStore) SaveSession(ctx context.Context, sess ClientSessionData) error { + m.lk.Lock() + defer m.lk.Unlock() + + m.sessions[memKey(sess.AccountDID, sess.SessionID)] = sess + return nil +} + +func (m *MemStore) DeleteSession(ctx context.Context, did syntax.DID, sessionID string) error { + m.lk.Lock() + defer m.lk.Unlock() + + delete(m.sessions, memKey(did, sessionID)) + return nil +} + +func (m *MemStore) GetAuthRequestInfo(ctx context.Context, state string) (*AuthRequestData, error) { + m.lk.Lock() + defer m.lk.Unlock() + + req, ok := m.requests[state] + if !ok { + return nil, fmt.Errorf("request info not found: %s", state) + } + // TODO: delete? should only ever fetch once + return &req, nil +} + +func (m *MemStore) SaveAuthRequestInfo(ctx context.Context, info AuthRequestData) error { + m.lk.Lock() + defer m.lk.Unlock() + + if _, ok := m.requests[info.State]; ok { + // Should be unreachable, barring implementation bugs elsewhere + return fmt.Errorf("auth request already saved for state %s", info.State) + } + + m.requests[info.State] = info + return nil +} + +func (m *MemStore) DeleteAuthRequestInfo(ctx context.Context, state string) error { + m.lk.Lock() + defer m.lk.Unlock() + + delete(m.requests, state) + return nil +} diff --git a/atproto/auth/oauth/oauth.go b/atproto/auth/oauth/oauth.go new file mode 100644 index 000000000..cd9324731 --- /dev/null +++ b/atproto/auth/oauth/oauth.go @@ -0,0 +1,724 @@ +package oauth + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/url" + "strings" + "time" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/go-querystring/query" +) + +var jwtExpirationDuration = 30 * time.Second + +// Service-level client. Used to establish and refrsh OAuth sessions, but is not itself account or session specific, and can not be used directly to make API calls on behalf of a user. +type ClientApp struct { + Client *http.Client + Resolver *Resolver + Dir identity.Directory + Config *ClientConfig + Store ClientAuthStore +} + +// App-level client configuration data. +// +// Not to be confused with the [ClientMetadata] struct type, which represents a full client metadata JSON document. +type ClientConfig struct { + // Full client identifier, which should be an HTTP URL + ClientID string + // Fully qualified callback URL + CallbackURL string + // Set of OAuth scope strings, which will be both declared in client metadata document and requested for every session. Must include "atproto". + Scopes []string + UserAgent string + + // For confidential clients, the private client assertion key. Note that while an interface is used here, only P-256 is allowed by the current specification. + PrivateKey atcrypto.PrivateKey + + // ID for current client assertion key (should be provided if PrivateKey is) + KeyID *string +} + +// Constructs a [ClientApp] based on configuration. +func NewClientApp(config *ClientConfig, store ClientAuthStore) *ClientApp { + app := &ClientApp{ + Client: http.DefaultClient, + Resolver: NewResolver(), + Dir: identity.DefaultDirectory(), + Config: config, + Store: store, + } + if config.UserAgent != "" { + app.Resolver.UserAgent = config.UserAgent + + // unpack DefaultDirectory nested type and insert UserAgent (and log failure in case default types change) + dirAgent := false + cdir, ok := app.Dir.(*identity.CacheDirectory) + if ok { + bdir, ok := cdir.Inner.(*identity.BaseDirectory) + if ok { + dirAgent = true + bdir.UserAgent = config.UserAgent + } + } + if !dirAgent { + slog.Info("OAuth ClientApp identity directory User-Agent not configured") + } + } + return app +} + +// Creates a basic [ClientConfig] for use as a public (non-confidential) client. To upgrade to a confidential client, use this method and then [ClientConfig.SetClientSecret]. +// +// The "scopes" array must include "atproto". +func NewPublicConfig(clientID, callbackURL string, scopes []string) ClientConfig { + c := ClientConfig{ + ClientID: clientID, + CallbackURL: callbackURL, + UserAgent: "indigo-sdk", + Scopes: scopes, + } + return c +} + +// Creates a basic [ClientConfig] for use with localhost developmnet. Such a client is always public (non-confidential). +// +// The "scopes" array must include "atproto". +func NewLocalhostConfig(callbackURL string, scopes []string) ClientConfig { + params := make(url.Values) + params.Set("redirect_uri", callbackURL) + params.Set("scope", scopeStr(scopes)) + c := ClientConfig{ + ClientID: fmt.Sprintf("http://localhost?%s", params.Encode()), + CallbackURL: callbackURL, + UserAgent: "indigo-sdk", + Scopes: scopes, + } + return c +} + +// Whether this is a "confidential" OAuth client (with configured client attestation key), versus "public" client. +func (config *ClientConfig) IsConfidential() bool { + return config.PrivateKey != nil && config.KeyID != nil +} + +// Set the secret key used for client assertions (for confidential clients). +// +// The corresponding public key (with matching key ID) must be present in the JWK list referenced by the client metadata document. +// +// Key IDs may be arbitrary strings such as UUIDs, stringified sequence numbers, or human-readable identifiers. +func (config *ClientConfig) SetClientSecret(priv atcrypto.PrivateKey, keyID string) error { + switch priv.(type) { + case *atcrypto.PrivateKeyP256: + // pass + case *atcrypto.PrivateKeyK256: + return fmt.Errorf("only P-256 (ES256) private keys supported for atproto OAuth") + default: + return fmt.Errorf("unknown private key type: %T", priv) + } + config.PrivateKey = priv + config.KeyID = &keyID + return nil +} + +// Returns a "JWKS" representation of public keys for the client. This can be returned as JSON, as part of client metadata. +// +// If the client does not have any keys (eg, public client), returns an empty set. +func (config *ClientConfig) PublicJWKS() JWKS { + + jwks := JWKS{Keys: []atcrypto.JWK{}} + + // public client with no keys + if config.PrivateKey == nil || config.KeyID == nil { + return jwks + } + + pub, err := config.PrivateKey.PublicKey() + if err != nil { + return jwks + } + jwk, err := pub.JWK() + if err != nil { + return jwks + } + jwk.KeyID = config.KeyID + + jwks.Keys = []atcrypto.JWK{*jwk} + return jwks +} + +// helper to turn a list of scope strings in to a single space-separated scope string +func scopeStr(scopes []string) string { + return strings.Join(scopes, " ") +} + +// Returns a [ClientMetadata] struct with the required fields populated based on this client configuration. Clients may want to populate additional metadata fields on top of this response. +// +// NOTE: confidential clients currently must provide JWKSURI after the fact +func (config *ClientConfig) ClientMetadata() ClientMetadata { + m := ClientMetadata{ + ClientID: config.ClientID, + ApplicationType: strPtr("web"), + GrantTypes: []string{"authorization_code", "refresh_token"}, + Scope: scopeStr(config.Scopes), + ResponseTypes: []string{"code"}, + RedirectURIs: []string{config.CallbackURL}, + DPoPBoundAccessTokens: true, + TokenEndpointAuthMethod: "none", + } + if config.IsConfidential() { + m.TokenEndpointAuthMethod = "private_key_jwt" + // NOTE: the key type is always ES256 + m.TokenEndpointAuthSigningAlg = strPtr("ES256") + + // TODO: need to include 'use' or 'key_ops' for JWKS in the client metadata doc? + //jwks := config.PublicJWKS() + //m.JWKS = &jwks + } + return m +} + +// High-level helper for fetching session data from store, based on account DID and session identifier. +func (app *ClientApp) ResumeSession(ctx context.Context, did syntax.DID, sessionID string) (*ClientSession, error) { + + sd, err := app.Store.GetSession(ctx, did, sessionID) + if err != nil { + return nil, err + } + + sess := ClientSession{ + Client: app.Client, + Config: app.Config, + Data: sd, + } + + // configure callback for updating session data + if app.Store != nil { + sess.PersistSessionCallback = func(ctx context.Context, data *ClientSessionData) { + slog.Debug("storing updated session data", "did", data.AccountDID, "session_id", data.SessionID) + err := app.Store.SaveSession(ctx, *data) + if err != nil { + slog.Error("failed to store updated session data", "did", data.AccountDID, "session_id", data.SessionID, "err", err) + } + } + } + + // TODO: refactor this in to ClientAuthStore layer? + priv, err := atcrypto.ParsePrivateMultibase(sd.DPoPPrivateKeyMultibase) + if err != nil { + return nil, err + } + sess.DPoPPrivateKey = priv + return &sess, nil +} + +type clientAssertionClaims struct { + jwt.RegisteredClaims + + HTTPMethod string `json:"htm"` + TargetURI string `json:"hti"` + AccessTokenHash *string `json:"ath,omitempty"` + Nonce *string `json:"nonce,omitempty"` +} + +type dpopClaims struct { + jwt.RegisteredClaims + + HTTPMethod string `json:"htm"` + TargetURI string `json:"htu"` + AccessTokenHash *string `json:"ath,omitempty"` + Nonce *string `json:"nonce,omitempty"` +} + +// Low-level helper to generate and sign an OAuth confidential client assertion token (JWT). +func (cfg *ClientConfig) NewClientAssertion(authURL string) (string, error) { + if !cfg.IsConfidential() { + return "", fmt.Errorf("non-confidential client") + } + claims := clientAssertionClaims{ + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: cfg.ClientID, + Subject: cfg.ClientID, + Audience: []string{authURL}, + ID: secureRandomBase64(16), + IssuedAt: jwt.NewNumericDate(time.Now()), + ExpiresAt: jwt.NewNumericDate(time.Now().Add(jwtExpirationDuration)), + }, + } + + signingMethod, err := keySigningMethod(cfg.PrivateKey) + if err != nil { + return "", err + } + + token := jwt.NewWithClaims(signingMethod, claims) + token.Header["kid"] = cfg.KeyID + return token.SignedString(cfg.PrivateKey) +} + +// Creates a DPoP token (JWT) for use with an OAuth Auth Server (not to be used with Resource Server). The returned JWT is not bound to an Access Token (no 'ath'), and does not indicate an issuer ('iss'). +// +// This is used during initial auth request (PAR), initial token request, and subsequent refresh token requests. Note that a full [ClientSession] is not available in several of these circumstances, so this is a stand-alone function. +func NewAuthDPoP(httpMethod, url, dpopNonce string, privKey atcrypto.PrivateKey) (string, error) { + + claims := dpopClaims{ + HTTPMethod: httpMethod, + TargetURI: url, + RegisteredClaims: jwt.RegisteredClaims{ + ID: secureRandomBase64(16), + IssuedAt: jwt.NewNumericDate(time.Now()), + ExpiresAt: jwt.NewNumericDate(time.Now().Add(jwtExpirationDuration)), + }, + } + if dpopNonce != "" { + claims.Nonce = &dpopNonce + } + + keyMethod, err := keySigningMethod(privKey) + if err != nil { + return "", err + } + + // TODO: parse/cache this public JWK, for efficiency + pub, err := privKey.PublicKey() + if err != nil { + return "", err + } + pubJWK, err := pub.JWK() + if err != nil { + return "", err + } + + token := jwt.NewWithClaims(keyMethod, claims) + token.Header["typ"] = "dpop+jwt" + token.Header["jwk"] = pubJWK + return token.SignedString(privKey) +} + +// attempts to read an HTTP response body as JSON, and determine an error reason. always closes the response body +func parseAuthErrorReason(resp *http.Response, reqType string) string { + defer resp.Body.Close() + var errResp map[string]any + if err := json.NewDecoder(resp.Body).Decode(&errResp); err != nil { + slog.Warn("auth server request failed", "request", reqType, "statusCode", resp.StatusCode, "err", err) + return "unknown" + } + slog.Warn("auth server request failed", "request", reqType, "statusCode", resp.StatusCode, "body", errResp) + return fmt.Sprintf("%s", errResp["error"]) +} + +// Low-level helper to send PAR request to auth server, which involves starting PKCE and DPoP. +func (app *ClientApp) SendAuthRequest(ctx context.Context, authMeta *AuthServerMetadata, scopes []string, loginHint string) (*AuthRequestData, error) { + + parURL := authMeta.PushedAuthorizationRequestEndpoint + state := secureRandomBase64(16) + pkceVerifier := secureRandomBase64(48) + + // generate PKCE code challenge for use in PAR request + codeChallenge := S256CodeChallenge(pkceVerifier) + + slog.Debug("preparing PAR", "client_id", app.Config.ClientID, "callback_url", app.Config.CallbackURL) + body := PushedAuthRequest{ + ClientID: app.Config.ClientID, + State: state, + RedirectURI: app.Config.CallbackURL, + Scope: scopeStr(scopes), + ResponseType: "code", + CodeChallenge: codeChallenge, + CodeChallengeMethod: "S256", + } + + if app.Config.IsConfidential() { + // self-signed JWT using private key in client metadata (confidential client) + assertionJWT, err := app.Config.NewClientAssertion(authMeta.Issuer) + if err != nil { + return nil, err + } + body.ClientAssertionType = ClientAssertionJWTBearer + body.ClientAssertion = assertionJWT + } + + if loginHint != "" { + body.LoginHint = &loginHint + } + vals, err := query.Values(body) + if err != nil { + return nil, err + } + bodyBytes := []byte(vals.Encode()) + + // when starting a new session, we don't know the DPoP nonce yet + dpopServerNonce := "" + + // create new key for the session + dpopPrivKey, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + return nil, err + } + + slog.Debug("sending auth request", "scopes", scopes, "state", state, "redirectURI", app.Config.CallbackURL) + + var resp *http.Response + for range 2 { + dpopJWT, err := NewAuthDPoP("POST", parURL, dpopServerNonce, dpopPrivKey) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, "POST", parURL, bytes.NewBuffer(bodyBytes)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("DPoP", dpopJWT) + + resp, err = app.Client.Do(req) + if err != nil { + return nil, err + } + + // update DPoP Nonce + dpopServerNonce = resp.Header.Get("DPoP-Nonce") + + // check for an error condition caused by an out of date DPoP nonce + // note that the HTTP status code would be 400 Bad Request on token endpoint, not 401 Unauthorized like it would be on Resource Server requests + if resp.StatusCode == http.StatusBadRequest && dpopServerNonce != "" { + // parseAuthErrorReason() always closes resp.Body + reason := parseAuthErrorReason(resp, "PAR") + if reason == "use_dpop_nonce" { + // already updated nonce value above; loop around and try again + continue + } + return nil, fmt.Errorf("PAR request failed (HTTP %d): %s", resp.StatusCode, reason) + } + + // otherwise process result + break + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + reason := parseAuthErrorReason(resp, "PAR") + return nil, fmt.Errorf("PAR request failed (HTTP %d): %s", resp.StatusCode, reason) + } + + var parResp PushedAuthResponse + if err := json.NewDecoder(resp.Body).Decode(&parResp); err != nil { + return nil, fmt.Errorf("auth request (PAR) response failed to decode: %w", err) + } + + parInfo := AuthRequestData{ + State: state, + AuthServerURL: authMeta.Issuer, + Scopes: scopes, + PKCEVerifier: pkceVerifier, + RequestURI: parResp.RequestURI, + AuthServerTokenEndpoint: authMeta.TokenEndpoint, + AuthServerRevocationEndpoint: authMeta.RevocationEndpoint, + DPoPAuthServerNonce: dpopServerNonce, + DPoPPrivateKeyMultibase: dpopPrivKey.Multibase(), + } + + return &parInfo, nil +} + +// Lower-level helper. This is usually invoked as part of [ClientApp.ProcessCallback]. +func (app *ClientApp) SendInitialTokenRequest(ctx context.Context, authCode string, info AuthRequestData) (*TokenResponse, error) { + + body := InitialTokenRequest{ + ClientID: app.Config.ClientID, + RedirectURI: app.Config.CallbackURL, + GrantType: "authorization_code", + Code: authCode, + CodeVerifier: info.PKCEVerifier, + } + + if app.Config.IsConfidential() { + clientAssertion, err := app.Config.NewClientAssertion(info.AuthServerURL) + if err != nil { + return nil, err + } + body.ClientAssertionType = &ClientAssertionJWTBearer + body.ClientAssertion = &clientAssertion + } + + dpopPrivKey, err := atcrypto.ParsePrivateMultibase(info.DPoPPrivateKeyMultibase) + if err != nil { + return nil, err + } + + vals, err := query.Values(body) + if err != nil { + return nil, err + } + bodyBytes := []byte(vals.Encode()) + + dpopServerNonce := info.DPoPAuthServerNonce + + var resp *http.Response + for range 2 { + dpopJWT, err := NewAuthDPoP("POST", info.AuthServerTokenEndpoint, dpopServerNonce, dpopPrivKey) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, "POST", info.AuthServerTokenEndpoint, bytes.NewBuffer(bodyBytes)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("DPoP", dpopJWT) + + resp, err = app.Client.Do(req) + if err != nil { + return nil, err + } + + // check if a nonce was provided + dpopNonceHdr := resp.Header.Get("DPoP-Nonce") + if dpopNonceHdr != "" && dpopNonceHdr != dpopServerNonce { + dpopServerNonce = dpopNonceHdr + } + + // check for an error condition caused by an out of date DPoP nonce + // note that the HTTP status code would be 400 Bad Request on token endpoint, not 401 Unauthorized like it would be on Resource Server requests + if resp.StatusCode == http.StatusBadRequest && dpopNonceHdr != "" { + // parseAuthErrorReason() always closes resp.Body + reason := parseAuthErrorReason(resp, "initial-token") + if reason == "use_dpop_nonce" { + // already updated nonce value above; loop around and try again + continue + } + return nil, fmt.Errorf("initial token request failed (HTTP %d): %s", resp.StatusCode, reason) + } + + // otherwise process result + break + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + reason := parseAuthErrorReason(resp, "initial-token") + return nil, fmt.Errorf("initial token request failed (HTTP %d): %s", resp.StatusCode, reason) + } + + var tokenResp TokenResponse + if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + return nil, fmt.Errorf("token response failed to decode: %w", err) + } + + return &tokenResp, nil +} + +// High-level helper for starting a new session. Resolves identifier to resource server and auth server metadata, sends PAR request, persists request info to store, and returns a redirect URL. +// +// The `identifier` argument can be an atproto account identifier (handle or DID), or can be a URL to the account's auth server. +// +// The returned sting will be a web URL that the user should be redirected to (in browser) to approve the auth flow. +func (app *ClientApp) StartAuthFlow(ctx context.Context, identifier string) (string, error) { + + var authserverURL string + var accountDID syntax.DID + + if strings.HasPrefix(identifier, "https://") { + authserverURL = identifier + identifier = "" + } else { + atid, err := syntax.ParseAtIdentifier(identifier) + if err != nil { + return "", fmt.Errorf("not a valid account identifier (%s): %w", identifier, err) + } + ident, err := app.Dir.Lookup(ctx, atid) + if err != nil { + return "", fmt.Errorf("failed to resolve username (%s): %w", identifier, err) + } + accountDID = ident.DID + host := ident.PDSEndpoint() + if host == "" { + return "", fmt.Errorf("identity does not link to an atproto host (PDS)") + } + + // TODO: logger on ClientApp? + logger := slog.Default().With("did", ident.DID, "handle", ident.Handle, "host", host) + logger.Debug("resolving to auth server metadata") + authserverURL, err = app.Resolver.ResolveAuthServerURL(ctx, host) + if err != nil { + return "", fmt.Errorf("resolving auth server: %w", err) + } + } + + authserverMeta, err := app.Resolver.ResolveAuthServerMetadata(ctx, authserverURL) + if err != nil { + return "", fmt.Errorf("fetching auth server metadata: %w", err) + } + + info, err := app.SendAuthRequest(ctx, authserverMeta, app.Config.Scopes, identifier) + if err != nil { + return "", fmt.Errorf("auth request failed: %w", err) + } + + if accountDID != "" { + info.AccountDID = &accountDID + } + + // persist auth request info + app.Store.SaveAuthRequestInfo(ctx, *info) + + params := url.Values{} + params.Set("client_id", app.Config.ClientID) + params.Set("request_uri", info.RequestURI) + + // AuthorizationEndpoint was already checked to be a clean URL + // TODO: could do additional SSRF checks on the redirect domain here + redirectURL := fmt.Sprintf("%s?%s", authserverMeta.AuthorizationEndpoint, params.Encode()) + return redirectURL, nil +} + +// High-level helper for completing auth flow: verifies callback query parameters against persisted auth request info, makes initial token request to the auth server, validates account identifier, and persists session data. +func (app *ClientApp) ProcessCallback(ctx context.Context, params url.Values) (*ClientSessionData, error) { + // There are two callback response formats, for error and non-error conditions, each expecting different + // parameters. + // + // Error responses expect: state, error (and optionally: error_description, error_uri) + // Non-error responses expect: state, iss, code + + state := params.Get("state") + if state == "" { + return nil, fmt.Errorf("missing state query param") + } + + info, err := app.Store.GetAuthRequestInfo(ctx, state) + if err != nil { + return nil, fmt.Errorf("loading auth request info: %w", err) + } + // This check should never fail, but it guards against a faulty ClientAuthStore implementation + if info.State != state { + return nil, fmt.Errorf("callback state doesn't match request info") + } + + // NOTE: A corresponding `state` is expected even under error conditions, + // hence we check error *after* checking state. + errorCode := params.Get("error") + if errorCode != "" { + var errorUri *syntax.URI + parsedUri, err := syntax.ParseURI(params.Get("error_uri")) + if err == nil { + errorUri = &parsedUri + } + return nil, &AuthRequestCallbackError{ + ErrorCode: errorCode, + ErrorDescription: params.Get("error_description"), + ErrorURI: errorUri, + } + } + + // If we reached here, there was no `error` and we can process the rest of the parameters + authserverURL := params.Get("iss") + authCode := params.Get("code") + if authserverURL == "" || authCode == "" { + return nil, fmt.Errorf("missing required query param") + } + + if info.AuthServerURL != authserverURL { + return nil, fmt.Errorf("callback iss doesn't match request info") + } + + tokenResp, err := app.SendInitialTokenRequest(ctx, authCode, *info) + if err != nil { + return nil, fmt.Errorf("initial token request: %w", err) + } + + // verify against account/server from start of login + var accountDID syntax.DID + var hostURL string + if info.AccountDID != nil { + // if we started with an account DID, verify it against the subject + accountDID = *info.AccountDID + if tokenResp.Subject != info.AccountDID.String() { + return nil, fmt.Errorf("token subject didn't match original DID") + } + // identity lookup for PDS hostname; this should be cached + ident, err := app.Dir.LookupDID(ctx, accountDID) + if err != nil { + return nil, err + } + hostURL = ident.PDSEndpoint() + } else { + // if we started with an auth server URL, resolve and verify the identity + accountDID, err = syntax.ParseDID(tokenResp.Subject) + if err != nil { + return nil, err + } + ident, err := app.Dir.LookupDID(ctx, accountDID) + if err != nil { + return nil, err + } + hostURL = ident.PDSEndpoint() + res, err := app.Resolver.ResolveAuthServerURL(ctx, hostURL) + if err != nil { + return nil, fmt.Errorf("resolving auth server: %w", err) + } + if res != authserverURL { + return nil, fmt.Errorf("token subject auth server did not match original") + } + } + + sessData := ClientSessionData{ + AccountDID: accountDID, + SessionID: info.State, + HostURL: hostURL, + AuthServerURL: info.AuthServerURL, + AuthServerTokenEndpoint: info.AuthServerTokenEndpoint, + AuthServerRevocationEndpoint: info.AuthServerRevocationEndpoint, + Scopes: strings.Split(tokenResp.Scope, " "), + AccessToken: tokenResp.AccessToken, + RefreshToken: tokenResp.RefreshToken, + DPoPAuthServerNonce: info.DPoPAuthServerNonce, + DPoPHostNonce: info.DPoPAuthServerNonce, // bootstrap host nonce from authserver + DPoPPrivateKeyMultibase: info.DPoPPrivateKeyMultibase, + } + if err := app.Store.SaveSession(ctx, sessData); err != nil { + return nil, err + } + if err := app.Store.DeleteAuthRequestInfo(ctx, state); err != nil { + // only log on failure to delete state info + slog.Warn("failed to delete auth request info", "state", state, "did", accountDID, "authserver", info.AuthServerURL, "err", err) + } + return &sessData, nil +} + +// High-level helper to delete a session, including revoking access/refresh tokens if supported by the AS +func (app *ClientApp) Logout(ctx context.Context, did syntax.DID, sessionID string) error { + sess, err := app.ResumeSession(ctx, did, sessionID) + if err != nil { + return err + } + + // Tell the AS to revoke the tokens, if supported + if sess.Data.AuthServerRevocationEndpoint == "" { + slog.Info("AS does not support token revocation, skipping RevokeSession") + } else { + err = sess.RevokeSession(ctx) + if err != nil { + slog.Warn("error during session revocation", "err", err) + } + } + + // Delete from our own session store + err = app.Store.DeleteSession(ctx, did, sessionID) + if err != nil { + return err + } + + return nil +} diff --git a/atproto/auth/oauth/resolver.go b/atproto/auth/oauth/resolver.go new file mode 100644 index 000000000..297c349db --- /dev/null +++ b/atproto/auth/oauth/resolver.go @@ -0,0 +1,170 @@ +package oauth + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/bluesky-social/indigo/util/ssrf" +) + +// Helper for resolving OAuth documents from the public web: client metadata, auth server metadata, etc. +// +// NOTE: configurable caching will likely be added in the future, but is not implemented yet. This struct may become an interface to support more flexible caching and resolution policies. +type Resolver struct { + Client *http.Client + UserAgent string +} + +func NewResolver() *Resolver { + c := http.Client{ + Timeout: 10 * time.Second, + Transport: ssrf.PublicOnlyTransport(), + } + return &Resolver{ + Client: &c, + UserAgent: "indigo-sdk", + } +} + +// Resolves a Resource Server URL (eg, an atproto account's registered PDS service URL) to an auth server URL (eg, entryway URL). They might be the same server! +// +// Ensures that the returned URL is valid (eg, parses as a URL). +func (r *Resolver) ResolveAuthServerURL(ctx context.Context, hostURL string) (string, error) { + u, err := url.Parse(hostURL) + if err != nil { + return "", err + } + // TODO: check against other resource server rules? + if u.Scheme != "https" || u.Hostname() == "" || u.Port() != "" { + return "", fmt.Errorf("not a valid public host URL: %s", hostURL) + } + + docURL := fmt.Sprintf("https://%s/.well-known/oauth-protected-resource", u.Hostname()) + + // NOTE: this allows redirects + req, err := http.NewRequestWithContext(ctx, "GET", docURL, nil) + if err != nil { + return "", err + } + if r.UserAgent != "" { + req.Header.Set("User-Agent", r.UserAgent) + } + + resp, err := r.Client.Do(req) + if err != nil { + return "", fmt.Errorf("fetching protected resource document: %w", err) + } + defer resp.Body.Close() + + // intentionally check for exactly HTTP 200 (not just 2xx) + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP error fetching protected resource document: %d", resp.StatusCode) + } + + var body ProtectedResourceMetadata + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return "", fmt.Errorf("invalid protected resource document: %w", err) + } + if len(body.AuthorizationServers) < 1 { + return "", fmt.Errorf("no auth server URL in protected resource document") + } + authURL := body.AuthorizationServers[0] + au, err := url.Parse(body.AuthorizationServers[0]) + if err != nil { + return "", fmt.Errorf("invalid auth server URL: %w", err) + } + if au.Scheme != "https" || au.Hostname() == "" || au.Port() != "" { + return "", fmt.Errorf("not a valid public auth server URL: %s", authURL) + } + return authURL, nil +} + +// Resolves an Auth Server URL to server metadata. Validates metadata before returning. +func (r *Resolver) ResolveAuthServerMetadata(ctx context.Context, serverURL string) (*AuthServerMetadata, error) { + u, err := url.Parse(serverURL) + if err != nil { + return nil, err + } + // TODO: check against other rules? + if u.Scheme != "https" || u.Hostname() == "" || u.Port() != "" { + return nil, fmt.Errorf("not a valid public host URL: %s", serverURL) + } + + docURL := fmt.Sprintf("https://%s/.well-known/oauth-authorization-server", u.Hostname()) + + // NOTE: this allows redirects + req, err := http.NewRequestWithContext(ctx, "GET", docURL, nil) + if err != nil { + return nil, err + } + if r.UserAgent != "" { + req.Header.Set("User-Agent", r.UserAgent) + } + + resp, err := r.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching auth server metadata: %w", err) + } + defer resp.Body.Close() + + // NOTE: maybe any HTTP 2xx should be allowed? + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP error fetching auth server metadata: %d", resp.StatusCode) + } + + var body AuthServerMetadata + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return nil, fmt.Errorf("invalid protected resource document: %w", err) + } + + if err := body.Validate(serverURL); err != nil { + return nil, err + } + return &body, nil +} + +// Fetches and validates OAuth client metadata document based on identifier in URL format. +func (r *Resolver) ResolveClientMetadata(ctx context.Context, clientID string) (*ClientMetadata, error) { + u, err := url.Parse(clientID) + if err != nil { + return nil, err + } + // TODO: check against other rules? + if u.Scheme != "https" || u.Hostname() == "" || u.Port() != "" { + return nil, fmt.Errorf("not a valid public host URL: %s", clientID) + } + + // NOTE: this allows redirects + req, err := http.NewRequestWithContext(ctx, "GET", clientID, nil) + if err != nil { + return nil, err + } + if r.UserAgent != "" { + req.Header.Set("User-Agent", r.UserAgent) + } + + resp, err := r.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching client metadata: %w", err) + } + defer resp.Body.Close() + + // NOTE: maybe any HTTP 2xx should be allowed? + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP error fetching auth server metadata: %d", resp.StatusCode) + } + + var body ClientMetadata + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return nil, fmt.Errorf("invalid client metadata document: %w", err) + } + + if err := body.Validate(clientID); err != nil { + return nil, err + } + return &body, nil +} diff --git a/atproto/auth/oauth/resolver_test.go b/atproto/auth/oauth/resolver_test.go new file mode 100644 index 000000000..9eb7be19b --- /dev/null +++ b/atproto/auth/oauth/resolver_test.go @@ -0,0 +1,149 @@ +package oauth + +import ( + "context" + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TODO: localhost (dev mode) resolution + +func TestValidateMetadata(t *testing.T) { + assert := assert.New(t) + + { + var meta ProtectedResourceMetadata + b, err := os.ReadFile("testdata/morel-protected-resource.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + } + + { + var meta ProtectedResourceMetadata + b, err := os.ReadFile("testdata/indie-protected-resource.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + } + + { + var meta AuthServerMetadata + b, err := os.ReadFile("testdata/bsky-entryway-authorization-server.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + assert.NoError(meta.Validate("https://bsky.social/.well-known/oauth-authorization-server")) + } + + { + var meta AuthServerMetadata + b, err := os.ReadFile("testdata/indie-authorization-server.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + assert.NoError(meta.Validate("https://pds.robocracy.org/.well-known/oauth-authorization-server")) + } + + { + var meta ClientMetadata + b, err := os.ReadFile("testdata/flaskdemo-client-metadata.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + assert.NoError(meta.Validate("https://oauth-flask.demo.bsky.dev/oauth/client-metadata.json")) + } + + { + var meta ClientMetadata + b, err := os.ReadFile("testdata/statusphere-client-metadata.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + assert.NoError(meta.Validate("https://statusphere.mozzius.dev/oauth-client-metadata.json")) + } + + { + var meta ClientMetadata + b, err := os.ReadFile("testdata/smokesignal-client-metadata.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + assert.NoError(meta.Validate("https://smokesignal.events/oauth/client-metadata.json")) + } + + { + var meta JWKS + b, err := os.ReadFile("testdata/flaskdemo-jwks.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + } + + { + var meta JWKS + b, err := os.ReadFile("testdata/smokesignal-jwks.json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &meta); err != nil { + t.Fatal(err) + } + } +} + +func TestResolver(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + resolver := NewResolver() + + { + // Live network tests (disabled by default) + /* + _, err := resolver.ResolveAuthServerURL(ctx, "https://morel.us-east.host.bsky.network") + assert.NoError(err) + _, err = resolver.ResolveAuthServerMetadata(ctx, "https://bsky.social") + assert.NoError(err) + _, err = resolver.ResolveClientMetadata(ctx, "https://oauth-flask.demo.bsky.dev/oauth/client-metadata.json") + assert.NoError(err) + */ + } + + { + // local unsafe should fail + _, err := resolver.ResolveAuthServerURL(ctx, "https://127.0.0.1") + assert.ErrorContains(err, "is not a public IP address") + _, err = resolver.ResolveAuthServerMetadata(ctx, "https://10.0.0.1") + assert.ErrorContains(err, "is not a public IP address") + _, err = resolver.ResolveClientMetadata(ctx, "https://127.0.0.1/oauth/client-metadata.json") + assert.ErrorContains(err, "is not a public IP address") + } +} diff --git a/atproto/auth/oauth/session.go b/atproto/auth/oauth/session.go new file mode 100644 index 000000000..f78fe0ecb --- /dev/null +++ b/atproto/auth/oauth/session.go @@ -0,0 +1,413 @@ +package oauth + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/go-querystring/query" +) + +type PersistSessionCallback = func(ctx context.Context, data *ClientSessionData) + +// Persisted information about an OAuth session. Used to resume an active session. +type ClientSessionData struct { + // Account DID for this session. Assuming only one active session per account, this can be used as "primary key" for storing and retrieving this information. + AccountDID syntax.DID `json:"account_did"` + + // Identifier to distinguish this particular session for the account. Server backends generally support multiple sessions for the same account. This package will re-use the random 'state' token from the auth flow as the session ID. + SessionID string `json:"session_id"` + + // Base URL of the "resource server" (eg, PDS). Should include scheme, hostname, port; no path or auth info. + HostURL string `json:"host_url"` + + // Base URL of the "auth server" (eg, PDS or entryway). Should include scheme, hostname, port; no path or auth info. + AuthServerURL string `json:"authserver_url"` + + // Full token endpoint + AuthServerTokenEndpoint string `json:"authserver_token_endpoint"` + + // Full revocation endpoint, if it exists + AuthServerRevocationEndpoint string `json:"authserver_revocation_endpoint,omitempty"` + + // The set of scopes approved for this session (returned in the initial token request) + Scopes []string `json:"scopes"` + + // Token which can be used directly against host ("resource server", eg PDS) + AccessToken string `json:"access_token"` + + // Token which can be sent to auth server (eg, PDS or entryway) to get a new access token + RefreshToken string `json:"refresh_token"` + + // Current auth server DPoP nonce + DPoPAuthServerNonce string `json:"dpop_authserver_nonce"` + + // Current host ("resource server", eg PDS) DPoP nonce + DPoPHostNonce string `json:"dpop_host_nonce"` + + // The secret cryptographic key generated by the client for this specific OAuth session + DPoPPrivateKeyMultibase string `json:"dpop_privatekey_multibase"` + + // TODO: also persist access token creation time / expiration time? In context that token might not be an easily parsed JWT +} + +// Implementation of [atclient.AuthMethod] for an OAuth session. Handles DPoP request token signing and nonce rotation, and token refresh requests. Optionally uses a callback to persist updated session data. +// +// A single ClientSession instance can be called concurrently: updates to session data (the 'Data' field) are protected with a RW mutex lock. Note that concurrent calls to distinct ClientSession instances for the same session could result in clobbered session data. +type ClientSession struct { + // HTTP client used for token refresh requests + Client *http.Client + + Config *ClientConfig + Data *ClientSessionData + DPoPPrivateKey atcrypto.PrivateKey + + PersistSessionCallback PersistSessionCallback + + // Lock which protects concurrent access to session data (eg, access and refresh tokens) + lk sync.RWMutex +} + +// Helper method to handle DPoP retries and client assertions (if the client is confidential) +// body object will be url-encoded (expected to be either RefreshTokenRequest or RevocationRequest) +// expects sess.lk to be held by caller +// if a non-nil *http.Response is returned, the caller is responsible for closing the response body +func (sess *ClientSession) postToAuthServer(ctx context.Context, url string, body interface{}) (*http.Response, error) { + vals, err := query.Values(body) + if err != nil { + return nil, err + } + if sess.Config.IsConfidential() { + clientAssertion, err := sess.Config.NewClientAssertion(sess.Data.AuthServerURL) + if err != nil { + return nil, err + } + vals.Set("client_assertion_type", ClientAssertionJWTBearer) + vals.Set("client_assertion", clientAssertion) + } + bodyBytes := []byte(vals.Encode()) + + var resp *http.Response + for range 2 { + dpopJWT, err := NewAuthDPoP("POST", url, sess.Data.DPoPAuthServerNonce, sess.DPoPPrivateKey) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(bodyBytes)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("DPoP", dpopJWT) + + resp, err = sess.Client.Do(req) + if err != nil { + return nil, err + } + + // always check if a new DPoP nonce was provided, and proactively update session data (even if there was not an explicit error) + dpopNonceHdr := resp.Header.Get("DPoP-Nonce") + if dpopNonceHdr != "" && dpopNonceHdr != sess.Data.DPoPAuthServerNonce { + sess.Data.DPoPAuthServerNonce = dpopNonceHdr + } + + // check for an error condition caused by an out of date DPoP nonce + // note that the HTTP status code is 400 Bad Request on the Auth Server token endpoint, not 401 Unauthorized like it would be on Resource Server requests + if resp.StatusCode == http.StatusBadRequest && dpopNonceHdr != "" { + // parseAuthErrorReason() always closes resp.Body + reason := parseAuthErrorReason(resp, "token-refresh") + if reason == "use_dpop_nonce" { + // already updated nonce value above; loop around and try again + continue + } + return nil, fmt.Errorf("auth server request failed (HTTP %d): %s", resp.StatusCode, reason) + } + + // otherwise process response (success or other error type) + break + } + + return resp, nil +} + +// Requests new tokens from auth server, and returns the new access token on success. +// +// Internally takes a lock on session data around the entire refresh process, including retries. Persists data using [PersistSessionCallback] if configured. +func (sess *ClientSession) RefreshTokens(ctx context.Context) (string, error) { + sess.lk.Lock() + defer sess.lk.Unlock() + + body := RefreshTokenRequest{ + ClientID: sess.Config.ClientID, + GrantType: "refresh_token", + RefreshToken: sess.Data.RefreshToken, + } + + resp, err := sess.postToAuthServer(ctx, sess.Data.AuthServerTokenEndpoint, body) + if err != nil { + return "", fmt.Errorf("token refresh failed: %w", err) + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + reason := parseAuthErrorReason(resp, "token-refresh") + return "", fmt.Errorf("token refresh failed (HTTP %d): %s", resp.StatusCode, reason) + } + + var tokenResp TokenResponse + if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + return "", fmt.Errorf("token response failed to decode: %w", err) + } + // TODO: more validation of token refresh response? + + sess.Data.AccessToken = tokenResp.AccessToken + sess.Data.RefreshToken = tokenResp.RefreshToken + + // persist updated data (tokens and possibly nonce) + if sess.PersistSessionCallback != nil { + sess.PersistSessionCallback(ctx, sess.Data) + } else { + slog.Warn("not saving updated session data", "did", sess.Data.AccountDID, "session_id", sess.Data.SessionID) + } + + return sess.Data.AccessToken, nil +} + +// If supported by the AS, use the revocation endpoint to revoke both the access token and the refresh token. +// This method always succeeds - any errors during revocation are logged but not returned. +func (sess *ClientSession) RevokeSession(ctx context.Context) error { + sess.lk.Lock() + defer sess.lk.Unlock() + + if sess.Data.AuthServerRevocationEndpoint == "" { + return fmt.Errorf("AS does not support token revocation") + } + + resp, err1 := sess.postToAuthServer(ctx, sess.Data.AuthServerRevocationEndpoint, RevocationRequest{ + ClientID: sess.Config.ClientID, + Token: sess.Data.AccessToken, + TokenTypeHint: "access_token", + }) + if err1 != nil { + err1 = fmt.Errorf("failed revoking access token: %w", err1) + } else { + if resp.StatusCode != http.StatusOK { + err1 = fmt.Errorf("bad HTTP status while revoking access token (%d)", resp.StatusCode) + } + resp.Body.Close() + } + + resp, err2 := sess.postToAuthServer(ctx, sess.Data.AuthServerRevocationEndpoint, RevocationRequest{ + ClientID: sess.Config.ClientID, + Token: sess.Data.RefreshToken, + TokenTypeHint: "refresh_token", + }) + if err2 != nil { + err2 = fmt.Errorf("failed revoking refresh token: %w", err1) + } else { + if resp.StatusCode != 200 { + err2 = fmt.Errorf("bad HTTP status while revoking refresh token (%d)", resp.StatusCode) + } + resp.Body.Close() + } + + return errors.Join(err1, err2) // returns nil if both errors are nil +} + +// Constructs and signs a DPoP JWT to include in request header to Host (aka Resource Server, aka PDS). These tokens are different from those used with Auth Server token endpoints (even if the PDS is filling both roles) +func (sess *ClientSession) NewHostDPoP(method, reqURL string) (string, error) { + sess.lk.RLock() + defer sess.lk.RUnlock() + + ath := S256CodeChallenge(sess.Data.AccessToken) + claims := dpopClaims{ + HTTPMethod: method, + TargetURI: reqURL, + AccessTokenHash: &ath, + RegisteredClaims: jwt.RegisteredClaims{ + Issuer: sess.Data.AuthServerURL, + ID: secureRandomBase64(16), + IssuedAt: jwt.NewNumericDate(time.Now()), + ExpiresAt: jwt.NewNumericDate(time.Now().Add(jwtExpirationDuration)), + }, + } + if sess.Data.DPoPHostNonce != "" { + claims.Nonce = &sess.Data.DPoPHostNonce + } + + keyMethod, err := keySigningMethod(sess.DPoPPrivateKey) + if err != nil { + return "", err + } + + // TODO: store a copy of this JWK on the ClientSession as a private field, for efficiency + pub, err := sess.DPoPPrivateKey.PublicKey() + if err != nil { + return "", err + } + pubJWK, err := pub.JWK() + if err != nil { + return "", err + } + + token := jwt.NewWithClaims(keyMethod, claims) + token.Header["typ"] = "dpop+jwt" + token.Header["jwk"] = pubJWK + return token.SignedString(sess.DPoPPrivateKey) +} + +// copy a request URL and strip query params and fragment, for DPoP +func dpopURL(u *url.URL) string { + u2 := *u + u2.RawQuery = "" + u2.ForceQuery = false + u2.Fragment = "" + u2.RawFragment = "" + return u2.String() +} + +// Parses a WWW-Authenticate response header to see if DPoP nonce update is indicated +func isNonceUpdateHeader(hdr string) bool { + // Example from RFC9449: + // WWW-Authenticate: DPoP error="use_dpop_nonce", error_description="Resource server requires nonce in DPoP proof" + return strings.Contains(hdr, "error=\"use_dpop_nonce\"") +} + +// Parses a WWW-Authenticate response header to see if access token has expired (needs refresh) +func isExpiredAccessTokenHeader(hdr string) bool { + // Example from OAuth 2.1 draft: + // WWW-Authenticate: Bearer error="invalid_token" error_description="The access token expired" + // TODO: should this also look for "expired"? + return strings.Contains(hdr, "error=\"invalid_token\"") +} + +func (sess *ClientSession) GetHostAccessData() (accessToken string, dpopHostNonce string) { + sess.lk.RLock() + defer sess.lk.RUnlock() + + return sess.Data.AccessToken, sess.Data.DPoPHostNonce +} + +func (sess *ClientSession) UpdateHostDPoPNonce(ctx context.Context, nonce string) { + sess.lk.Lock() + defer sess.lk.Unlock() + + sess.Data.DPoPHostNonce = nonce + + if sess.PersistSessionCallback != nil { + sess.PersistSessionCallback(ctx, sess.Data) + } else { + slog.Warn("not saving updated host DPoP nonce", "did", sess.Data.AccountDID, "session_id", sess.Data.SessionID) + } +} + +// Sends API request to OAuth Resource Server (PDS), using access token and DPoP. +// +// Automatically handles DPoP nonce updates and token refresh as needed, based on the response status code and `WWW-Authenticate` header. +func (sess *ClientSession) DoWithAuth(c *http.Client, req *http.Request, endpoint syntax.NSID) (*http.Response, error) { + + durl := dpopURL(req.URL) + + accessToken, dpopNonce := sess.GetHostAccessData() + + // this method may need to retry twice, once for DPoP nonce update and once for token refresh + var resp *http.Response + for range 3 { + dpopJWT, err := sess.NewHostDPoP(req.Method, durl) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("DPoP %s", accessToken)) + req.Header.Set("DPoP", dpopJWT) + + resp, err = c.Do(req) + if err != nil { + return nil, err + } + + // on Success, or many types of error, just return HTTP response + // "Unauthorized" is HTTP status code 401 + if resp.StatusCode != http.StatusUnauthorized || resp.Header.Get("WWW-Authenticate") == "" { + return resp, nil + } + + authHdr := resp.Header.Get("WWW-Authenticate") + dpopNonceHdr := resp.Header.Get("DPoP-Nonce") + + // if DPoP nonce changed, update and retry request + if isNonceUpdateHeader(authHdr) && dpopNonceHdr != "" { + // TODO: validate or normalize dpopNonceHdr in some way? eg minimum length + if dpopNonceHdr == dpopNonce { + return nil, fmt.Errorf("OAuth PDS DPoP nonce failure, but no new nonce supplied") + } + + // persist new nonce value via callback + sess.UpdateHostDPoPNonce(req.Context(), dpopNonceHdr) + dpopNonce = dpopNonceHdr + + // retry request + retry := req.Clone(req.Context()) + if req.GetBody != nil { + retry.Body, err = req.GetBody() + if err != nil { + return nil, fmt.Errorf("GetBody failed when retrying API request: %w", err) + } + } + req = retry + continue + } + + // if access token expired, refresh and retry + if isExpiredAccessTokenHeader(authHdr) { + accessToken, err = sess.RefreshTokens(req.Context()) + if err != nil { + return nil, fmt.Errorf("failed to refresh OAuth tokens: %w", err) + } + + retry := req.Clone(req.Context()) + if req.GetBody != nil { + retry.Body, err = req.GetBody() + if err != nil { + return nil, fmt.Errorf("GetBody failed when retrying API request: %w", err) + } + } + req = retry + continue + } + + // otherwise, this was some other type of auth failure; just return the full response + // NOTE: in theory we could return an APIError here instead + return resp, nil + } + + return nil, fmt.Errorf("OAuth client ran out of request retries") +} + +// Creates a new [atclient.APIClient] which wraps this session for auth. +func (sess *ClientSession) APIClient() *atclient.APIClient { + c := atclient.APIClient{ + Client: sess.Client, + Host: sess.Data.HostURL, + Auth: sess, + AccountDID: &sess.Data.AccountDID, + } + if sess.Config.UserAgent != "" { + c.Headers = make(map[string][]string) + c.Headers.Set("User-Agent", sess.Config.UserAgent) + } + return &c +} diff --git a/atproto/auth/oauth/store.go b/atproto/auth/oauth/store.go new file mode 100644 index 000000000..c054cdb80 --- /dev/null +++ b/atproto/auth/oauth/store.go @@ -0,0 +1,28 @@ +package oauth + +import ( + "context" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Interface for persisting session data and auth request data, required as part of an OAuth client app. +// +// This interface supports multiple sessions for a single account (DID). This is helpful for traditional web app backends where a single user might log in and have concurrent sessions from multiple browsers/devices. For situations where multiple sessions are not required, implementations of this interface could ignore the `sessionID` parameters, though this could result in clobbering of active sessions. +// +// For authentication-only (authn-only) applications, the `SaveSession()` method could be a no-op. +// +// Implementations should generally allow for concurrent access. +// +// `SaveSession()` should be treated as an "upsert" operation (updating a previously saved session with matching did and sessionID, if present). `SaveAuthRequestInfo()` however is create-only. +// +// Implementations are responsible for garbage-collecting expired sessions and auth requests. +type ClientAuthStore interface { + GetSession(ctx context.Context, did syntax.DID, sessionID string) (*ClientSessionData, error) + SaveSession(ctx context.Context, sess ClientSessionData) error + DeleteSession(ctx context.Context, did syntax.DID, sessionID string) error + + GetAuthRequestInfo(ctx context.Context, state string) (*AuthRequestData, error) + SaveAuthRequestInfo(ctx context.Context, info AuthRequestData) error + DeleteAuthRequestInfo(ctx context.Context, state string) error +} diff --git a/atproto/auth/oauth/testdata/bsky-entryway-authorization-server.json b/atproto/auth/oauth/testdata/bsky-entryway-authorization-server.json new file mode 100644 index 000000000..ec9419e89 --- /dev/null +++ b/atproto/auth/oauth/testdata/bsky-entryway-authorization-server.json @@ -0,0 +1 @@ +{"issuer":"https://bsky.social","request_parameter_supported":true,"request_uri_parameter_supported":true,"require_request_uri_registration":true,"scopes_supported":["atproto","transition:generic","transition:chat.bsky"],"subject_types_supported":["public"],"response_types_supported":["code"],"response_modes_supported":["query","fragment","form_post"],"grant_types_supported":["authorization_code","refresh_token"],"code_challenge_methods_supported":["S256"],"ui_locales_supported":["en-US"],"display_values_supported":["page","popup","touch"],"request_object_signing_alg_values_supported":["RS256","RS384","RS512","PS256","PS384","PS512","ES256","ES256K","ES384","ES512","none"],"authorization_response_iss_parameter_supported":true,"request_object_encryption_alg_values_supported":[],"request_object_encryption_enc_values_supported":[],"jwks_uri":"https://bsky.social/oauth/jwks","authorization_endpoint":"https://bsky.social/oauth/authorize","token_endpoint":"https://bsky.social/oauth/token","token_endpoint_auth_methods_supported":["none","private_key_jwt"],"token_endpoint_auth_signing_alg_values_supported":["RS256","RS384","RS512","PS256","PS384","PS512","ES256","ES256K","ES384","ES512"],"revocation_endpoint":"https://bsky.social/oauth/revoke","introspection_endpoint":"https://bsky.social/oauth/introspect","pushed_authorization_request_endpoint":"https://bsky.social/oauth/par","require_pushed_authorization_requests":true,"dpop_signing_alg_values_supported":["RS256","RS384","RS512","PS256","PS384","PS512","ES256","ES256K","ES384","ES512"],"client_id_metadata_document_supported":true} \ No newline at end of file diff --git a/atproto/auth/oauth/testdata/flaskdemo-client-metadata.json b/atproto/auth/oauth/testdata/flaskdemo-client-metadata.json new file mode 100644 index 000000000..ab75b4ec1 --- /dev/null +++ b/atproto/auth/oauth/testdata/flaskdemo-client-metadata.json @@ -0,0 +1 @@ +{"application_type":"web","client_id":"https://oauth-flask.demo.bsky.dev/oauth/client-metadata.json","client_name":"atproto OAuth Flask Backend Demo","client_uri":"https://oauth-flask.demo.bsky.dev/","dpop_bound_access_tokens":true,"grant_types":["authorization_code","refresh_token"],"jwks_uri":"https://oauth-flask.demo.bsky.dev/oauth/jwks.json","redirect_uris":["https://oauth-flask.demo.bsky.dev/oauth/callback"],"response_types":["code"],"scope":"atproto transition:generic","token_endpoint_auth_method":"private_key_jwt","token_endpoint_auth_signing_alg":"ES256"} diff --git a/atproto/auth/oauth/testdata/flaskdemo-jwks.json b/atproto/auth/oauth/testdata/flaskdemo-jwks.json new file mode 100644 index 000000000..8f99fbb56 --- /dev/null +++ b/atproto/auth/oauth/testdata/flaskdemo-jwks.json @@ -0,0 +1 @@ +{"keys":[{"crv":"P-256","kid":"demo-1723096995","kty":"EC","x":"9xgcqvuBJIN8M32cvXc7vZDc4xgaYvrEMh8LHH1Uz0E","y":"ar8LYDXhUp32aNjq-Ko5jKVFUZNLSYxm7okrU2KIUqk"}]} diff --git a/atproto/auth/oauth/testdata/indie-authorization-server.json b/atproto/auth/oauth/testdata/indie-authorization-server.json new file mode 100644 index 000000000..15212ee1a --- /dev/null +++ b/atproto/auth/oauth/testdata/indie-authorization-server.json @@ -0,0 +1 @@ +{"issuer":"https://pds.robocracy.org","request_parameter_supported":true,"request_uri_parameter_supported":true,"require_request_uri_registration":true,"scopes_supported":["atproto","transition:generic","transition:chat.bsky"],"subject_types_supported":["public"],"response_types_supported":["code"],"response_modes_supported":["query","fragment","form_post"],"grant_types_supported":["authorization_code","refresh_token"],"code_challenge_methods_supported":["S256"],"ui_locales_supported":["en-US"],"display_values_supported":["page","popup","touch"],"request_object_signing_alg_values_supported":["RS256","RS384","RS512","PS256","PS384","PS512","ES256","ES256K","ES384","ES512","none"],"authorization_response_iss_parameter_supported":true,"request_object_encryption_alg_values_supported":[],"request_object_encryption_enc_values_supported":[],"jwks_uri":"https://pds.robocracy.org/oauth/jwks","authorization_endpoint":"https://pds.robocracy.org/oauth/authorize","token_endpoint":"https://pds.robocracy.org/oauth/token","token_endpoint_auth_methods_supported":["none","private_key_jwt"],"token_endpoint_auth_signing_alg_values_supported":["RS256","RS384","RS512","PS256","PS384","PS512","ES256","ES256K","ES384","ES512"],"revocation_endpoint":"https://pds.robocracy.org/oauth/revoke","introspection_endpoint":"https://pds.robocracy.org/oauth/introspect","pushed_authorization_request_endpoint":"https://pds.robocracy.org/oauth/par","require_pushed_authorization_requests":true,"dpop_signing_alg_values_supported":["RS256","RS384","RS512","PS256","PS384","PS512","ES256","ES256K","ES384","ES512"],"protected_resources":["https://pds.robocracy.org"],"client_id_metadata_document_supported":true} \ No newline at end of file diff --git a/atproto/auth/oauth/testdata/indie-protected-resource.json b/atproto/auth/oauth/testdata/indie-protected-resource.json new file mode 100644 index 000000000..163ecb8d9 --- /dev/null +++ b/atproto/auth/oauth/testdata/indie-protected-resource.json @@ -0,0 +1 @@ +{"resource":"https://pds.robocracy.org","authorization_servers":["https://pds.robocracy.org"],"scopes_supported":[],"bearer_methods_supported":["header"],"resource_documentation":"https://atproto.com"} \ No newline at end of file diff --git a/atproto/auth/oauth/testdata/morel-protected-resource.json b/atproto/auth/oauth/testdata/morel-protected-resource.json new file mode 100644 index 000000000..c12d2cd59 --- /dev/null +++ b/atproto/auth/oauth/testdata/morel-protected-resource.json @@ -0,0 +1 @@ +{"resource":"https://morel.us-east.host.bsky.network","authorization_servers":["https://bsky.social"],"scopes_supported":[],"bearer_methods_supported":["header"],"resource_documentation":"https://atproto.com"} \ No newline at end of file diff --git a/atproto/auth/oauth/testdata/smokesignal-client-metadata.json b/atproto/auth/oauth/testdata/smokesignal-client-metadata.json new file mode 100644 index 000000000..81402d26c --- /dev/null +++ b/atproto/auth/oauth/testdata/smokesignal-client-metadata.json @@ -0,0 +1 @@ +{"client_id":"https://smokesignal.events/oauth/client-metadata.json","dpop_bound_access_tokens":true,"application_type":"web","redirect_uris":["https://smokesignal.events/oauth/callback"],"client_uri":"https://smokesignal.events","subject_type":"public","grant_types":["authorization_code","refresh_token"],"response_types":["code"],"scope":"atproto transition:generic","client_name":"Smoke Signal","token_endpoint_auth_method":"none","jwks_uri":"https://smokesignal.events/oauth/jwks.json","logo_uri":"https://smokesignal.events/static/logo-160x160x.png","tos_uri":"https://docs.smokesignal.events/docs/about/terms/","policy_uri":"https://docs.smokesignal.events/docs/about/privacy/"} \ No newline at end of file diff --git a/atproto/auth/oauth/testdata/smokesignal-jwks.json b/atproto/auth/oauth/testdata/smokesignal-jwks.json new file mode 100644 index 000000000..d17e20585 --- /dev/null +++ b/atproto/auth/oauth/testdata/smokesignal-jwks.json @@ -0,0 +1 @@ +{"keys":[{"kid":"01J5P69KNBQ2D08GBND5REQ5X6","alg":"ES256","kty":"EC","crv":"P-256","x":"1-Phkmhnqd8OzMezLc_iPeNFjyhVZtpm85FliOjbuRI","y":"7qxZDJ9UVBOR2AcF1IUSnD784vlVeoSA18O2XDGxL4U"},{"kid":"01J5P69N6XZTDEHMSBH8VJPYTV","alg":"ES256","kty":"EC","crv":"P-256","x":"xzk6X6PQ-r5jCoEPwAbBDO0bvG6Zy5TQWSDNLox6eQg","y":"LXg2ubRJ6uiqTNYQ9Pns1wEmtxyGAwUokTj5O3Ws92E"}]} \ No newline at end of file diff --git a/atproto/auth/oauth/testdata/statusphere-client-metadata.json b/atproto/auth/oauth/testdata/statusphere-client-metadata.json new file mode 100644 index 000000000..2f8c2fdc0 --- /dev/null +++ b/atproto/auth/oauth/testdata/statusphere-client-metadata.json @@ -0,0 +1 @@ +{"redirect_uris":["https://statusphere.mozzius.dev/oauth/callback"],"response_types":["code"],"grant_types":["authorization_code","refresh_token"],"scope":"atproto transition:generic","token_endpoint_auth_method":"none","application_type":"web","client_id":"https://statusphere.mozzius.dev/oauth-client-metadata.json","client_name":"Statusphere React App","client_uri":"https://statusphere.mozzius.dev","dpop_bound_access_tokens":true} \ No newline at end of file diff --git a/atproto/auth/oauth/types.go b/atproto/auth/oauth/types.go new file mode 100644 index 000000000..00df936fa --- /dev/null +++ b/atproto/auth/oauth/types.go @@ -0,0 +1,455 @@ +package oauth + +import ( + "errors" + "fmt" + "net/url" + "slices" + "strings" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +var ClientAssertionJWTBearer string = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + +var ( + ErrInvalidAuthServerMetadata = errors.New("invalid auth server metadata") + ErrInvalidClientMetadata = errors.New("invalid client metadata doc") +) + +type JWKS struct { + Keys []atcrypto.JWK `json:"keys"` +} + +// Expected response type from looking up OAuth Protected Resource information on a server (eg, a PDS instance) +type ProtectedResourceMetadata struct { + // are there other fields worth including? + + AuthorizationServers []string `json:"authorization_servers"` +} + +type ClientMetadata struct { + // Must exactly match the full URL used to fetch the client metadata file itself + ClientID string `json:"client_id"` + + // Must be one of `web` or `native`, with `web` as the default if not specified. + ApplicationType *string `json:"application_type,omitempty"` + + // `authorization_code` must always be included. `refresh_token` is optional, but must be included if the client will make token refresh requests. + GrantTypes []string `json:"grant_types"` + + // All scope values which might be requested by the client are declared here. The `atproto` scope is required, so must be included here. + Scope string `json:"scope"` + + // `code` must be included + ResponseTypes []string `json:"response_types"` + + // At least one redirect URI is required. + RedirectURIs []string `json:"redirect_uris"` + + // Confidential clients must set this to `private_key_jwt`; public must be `none`. + // In some sense this field is "optional" (including in atproto OAuth specs), but it is effectively required, because the default value is invalid for atproto OAuth. + TokenEndpointAuthMethod string `json:"token_endpoint_auth_method"` + + // `none` is never allowed here. The current recommended and most-supported algorithm is ES256, but this may evolve over time. + TokenEndpointAuthSigningAlg *string `json:"token_endpoint_auth_signing_alg,omitempty"` + + // DPoP is mandatory for all clients, so this must be present and true + DPoPBoundAccessTokens bool `json:"dpop_bound_access_tokens"` + + // confidential clients must supply at least one public key in JWK format for use with JWT client authentication. Either this field or the `jwks_uri` field must be provided for confidential clients, but not both. + JWKS *JWKS `json:"jwks,omitempty"` + + // URL pointing to a JWKS JSON object. See `jwks` above for details. + JWKSURI *string `json:"jwks_uri,omitempty"` + + // human-readable name of the client + ClientName *string `json:"client_name,omitempty"` + + // not to be confused with client_id, this is a homepage URL for the client. If provided, the client_uri must have the same hostname as client_id. + ClientURI *string `json:"client_uri,omitempty"` + + // URL to client logo. Only https: URIs are allowed. + LogoURI *string `json:"logo_uri,omitempty"` + + // URL to human-readable terms of service (ToS) for the client. Only https: URIs are allowed. + TosURI *string `json:"tos_uri,omitempty"` + + // URL to human-readable privacy policy for the client. Only https: URIs are allowed. + PolicyURI *string `json:"policy_uri,omitempty"` +} + +// returns 'true' if client metadata indicates that this is a confidential client +func (m *ClientMetadata) IsConfidential() bool { + if (m.JWKSURI != nil || (m.JWKS != nil && len(m.JWKS.Keys) > 0)) && m.TokenEndpointAuthMethod == "private_key_jwt" { + return true + } + + return false +} + +func (m *ClientMetadata) Validate(clientID string) error { + + if m.ClientID == "" || m.ClientID != clientID { + return fmt.Errorf("%w: client_id", ErrInvalidClientMetadata) + } + + if m.ApplicationType != nil && !slices.Contains([]string{"web", "native"}, *m.ApplicationType) { + return fmt.Errorf("%w: application_type must be 'web', 'native', or undefined", ErrInvalidClientMetadata) + } + + if !slices.Contains(m.GrantTypes, "authorization_code") { + return fmt.Errorf("%w: grant_type must include 'authorization_code'", ErrInvalidClientMetadata) + } + + scopes := strings.Split(m.Scope, " ") + if !slices.Contains(scopes, "atproto") { + return fmt.Errorf("%w: scope must include 'atproto'", ErrInvalidClientMetadata) + } + + if !slices.Contains(m.ResponseTypes, "code") { + return fmt.Errorf("%w: response_types must include 'code'", ErrInvalidClientMetadata) + } + + if len(m.RedirectURIs) == 0 { + return fmt.Errorf("%w: redirect_uris must have at least one element", ErrInvalidClientMetadata) + } + + // 'web' redirect URLs have more restrictions + if m.ApplicationType == nil || *m.ApplicationType == "web" { + for _, ru := range m.RedirectURIs { + u, err := url.Parse(ru) + if err != nil { + return fmt.Errorf("%w: invalid web redirect_uris: %w", ErrInvalidClientMetadata, err) + } + if u.Scheme != "https" && u.Hostname() != "127.0.0.1" { + return fmt.Errorf("%w: web redirect_uris must have 'https' scheme", ErrInvalidClientMetadata) + } + } + } + + if !(m.TokenEndpointAuthMethod == "none" || m.TokenEndpointAuthMethod == "private_key_jwt") { + return fmt.Errorf("%w: unsupported token_endpoint_auth_method", ErrInvalidClientMetadata) + } + + if m.TokenEndpointAuthSigningAlg != nil && *m.TokenEndpointAuthSigningAlg == "none" { + // NOTE: what if this is a public client? + return fmt.Errorf("%w: token_endpoint_auth_signing_alg must not be 'none'", ErrInvalidClientMetadata) + } + + if !m.DPoPBoundAccessTokens { + return fmt.Errorf("%w: dpop_bound_access_tokens must be true (DPoP is required)", ErrInvalidClientMetadata) + } + + if m.JWKSURI != nil && *m.JWKSURI == "" { + return fmt.Errorf("%w: jwks_uri must be valid URL (when provided)", ErrInvalidClientMetadata) + } + + // NOTE: metadata URLs are not validated (they are not an error for overall metadata doc) + + return nil +} + +type AuthServerMetadata struct { + + // the "origin" URL of the Authorization Server. Must be a valid URL, with https scheme. A port number is allowed (if that matches the origin), but the default port (443 for HTTPS) must not be specified. There must be no path segments. Must match the origin of the URL used to fetch the metadata document itself. + Issuer string `json:"issuer"` + + // endpoint URL for authorization redirects + AuthorizationEndpoint string `json:"authorization_endpoint"` + + // endpoint URL for token requests + TokenEndpoint string `json:"token_endpoint"` + + // must include code + ResponseTypesSupported []string `json:"response_types_supported"` + + // must include authorization_code and refresh_token (refresh tokens must be supported) + GrantTypesSupported []string `json:"grant_types_supported"` + + // must include S256 + CodeChallengeMethodsSupported []string `json:"code_challenge_methods_supported"` + + // must include both none (public clients) and private_key_jwt (confidential clients) + TokenEndpointAuthMethodsSupoorted []string `json:"token_endpoint_auth_methods_supported"` + + // must not include `none`. Must include ES256 for now. + TokenEndpointAuthSigningAlgValuesSupported []string `json:"token_endpoint_auth_signing_alg_values_supported"` + + // must include atproto. If supporting the transitional grants, they should be included here as well. + ScopesSupported []string `json:"scopes_supported"` + + // must be true + AuthorizationReponseISSParameterSupported bool `json:"authorization_response_iss_parameter_supported"` + + // must be true + RequirePushedAuthorizationRequests bool `json:"require_pushed_authorization_requests"` + + // corresponds to the PAR endpoint URL + PushedAuthorizationRequestEndpoint string `json:"pushed_authorization_request_endpoint"` + + // currently must include ES256 + DPoPSigningAlgValuesSupported []string `json:"dpop_signing_alg_values_supported"` + + // default is true; does not need to be set explicitly, but must not be false + RequireRequestURIRegistration *bool `json:"require_request_uri_registration,omitempty"` + + // must be true + ClientIDMetadataDocumentSupported bool `json:"client_id_metadata_document_supported"` + + // optional, used to explicitly revoke access/refresh tokens on logout, if present + RevocationEndpoint string `json:"revocation_endpoint,omitempty"` +} + +func (m *AuthServerMetadata) Validate(serverURL string) error { + + if m.Issuer == "" { + return fmt.Errorf("%w: empty issuer", ErrInvalidAuthServerMetadata) + } + u, err := url.Parse(m.Issuer) + if err != nil { + return fmt.Errorf("%w: invalid issuer URL: %w", ErrInvalidAuthServerMetadata, err) + } + if u.Scheme != "https" || u.Port() != "" || u.Path != "" || u.Fragment != "" || u.RawQuery != "" { + return fmt.Errorf("%w: issuer URL", ErrInvalidAuthServerMetadata) + } + + // check that Issuer matches domain this metadata document was fetched from + srvu, err := url.Parse(serverURL) + if err != nil { + return fmt.Errorf("%w: invalid request URL: %w", ErrInvalidAuthServerMetadata, err) + } + if u.Scheme != srvu.Scheme || u.Host != srvu.Host { + return fmt.Errorf("%w: issuer must match request URL", ErrInvalidAuthServerMetadata) + } + + // check that authorization endpoint is a valid HTTPS URL with no fragment or query params (we will be appending query params latter) + aeurl, err := url.Parse(m.AuthorizationEndpoint) + if err != nil { + return fmt.Errorf("%w: invalid auth endpoint URL (%s): %w", ErrInvalidAuthServerMetadata, m.AuthorizationEndpoint, err) + } + if aeurl.Scheme != "https" || u.Fragment != "" || u.RawQuery != "" { + return fmt.Errorf("%w: invalid auth endpoint URL: %s", ErrInvalidAuthServerMetadata, m.AuthorizationEndpoint) + } + + if !slices.Contains(m.ResponseTypesSupported, "code") { + return fmt.Errorf("%w: response_types_supported must include 'code'", ErrInvalidAuthServerMetadata) + } + if !slices.Contains(m.GrantTypesSupported, "authorization_code") { + return fmt.Errorf("%w: grant_types_supported must include 'authorization_code'", ErrInvalidAuthServerMetadata) + } + if !slices.Contains(m.GrantTypesSupported, "refresh_token") { + return fmt.Errorf("%w: grant_types_supported must include 'refresh_token'", ErrInvalidAuthServerMetadata) + } + if !slices.Contains(m.CodeChallengeMethodsSupported, "S256") { + return fmt.Errorf("%w: code_challenge_method must include 'S256'", ErrInvalidAuthServerMetadata) + } + if !slices.Contains(m.TokenEndpointAuthMethodsSupoorted, "none") { + return fmt.Errorf("%w: token_endpoint_auth_methods_supported must include 'none'", ErrInvalidAuthServerMetadata) + } + if !slices.Contains(m.TokenEndpointAuthMethodsSupoorted, "private_key_jwt") { + return fmt.Errorf("%w: token_endpoint_auth_methods_supported must include 'private_key_jwt'", ErrInvalidAuthServerMetadata) + } + if !slices.Contains(m.TokenEndpointAuthSigningAlgValuesSupported, "ES256") { + return fmt.Errorf("%w: token_endpoint_auth_signing_alg_values_supported must include 'ES256'", ErrInvalidAuthServerMetadata) + } + if !slices.Contains(m.ScopesSupported, "atproto") { + return fmt.Errorf("%w: scopes_supported must include 'atproto'", ErrInvalidAuthServerMetadata) + } + if !m.AuthorizationReponseISSParameterSupported { + return fmt.Errorf("%w: authorization_response_iss_parameter_supported must be true", ErrInvalidAuthServerMetadata) + } + if !m.RequirePushedAuthorizationRequests { + return fmt.Errorf("%w: require_pushed_authorization_requests must be true", ErrInvalidAuthServerMetadata) + } + if m.PushedAuthorizationRequestEndpoint == "" { + return fmt.Errorf("%w: pushed_authorization_request_endpoint is required", ErrInvalidAuthServerMetadata) + } + if !slices.Contains(m.DPoPSigningAlgValuesSupported, "ES256") { + return fmt.Errorf("%w: dpop_signing_alg_values_supported must include 'ES256'", ErrInvalidAuthServerMetadata) + } + if m.RequireRequestURIRegistration != nil && *m.RequireRequestURIRegistration != true { + return fmt.Errorf("%w: require_request_uri_registration must be undefined or true", ErrInvalidAuthServerMetadata) + } + if !m.ClientIDMetadataDocumentSupported { + return fmt.Errorf("%w: client_id_metadata_document_supported must be true", ErrInvalidAuthServerMetadata) + } + return nil +} + +// The fields which are included in a PAR request. These HTTP POST bodies are form-encoded, so use URL encoding syntax, not JSON. +type PushedAuthRequest struct { + // Client ID, aka client metadata URL + ClientID string `url:"client_id"` + + // Random identifier for this request, generated by client + State string `url:"state"` + + // Client-specified URL that will get redirected to by auth server at end of user auth flow + RedirectURI string `url:"redirect_uri"` + + // Requested auth scopes, as a space-delimited list + Scope string `url:"scope"` + + // Optional account identifier (DID or handle) to help with user account login and/or account switching + LoginHint *string `url:"login_hint,omitempty"` + + // Optional hint to auth server of what expected auth behavior should be. Eg, 'create', 'none', 'consent', 'login', 'select_account' + Prompt *string `url:"prompt,omitempty"` + + // Always "code" + ResponseType string `url:"response_type"` + + // Always "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ClientAssertionType string `url:"client_assertion_type"` + + // Confidential client signed JWT + ClientAssertion string `url:"client_assertion"` + + // Client-generated PKCE challenge hash, derived from random "verifier" string + CodeChallenge string `url:"code_challenge"` + + // Almost always "S256" + CodeChallengeMethod string `url:"code_challenge_method"` +} + +type PushedAuthResponse struct { + // unique token in URI format, which will be used by the client in the auth flow redirect + RequestURI string `json:"request_uri"` + + // positive integer indicating number of seconds the `request_uri` is valid for. + ExpiresIn int `json:"expires_in"` +} + +// Persisted information about an OAuth Auth Request. +type AuthRequestData struct { + // The random identifier generated by the client for the auth request flow. Can be used as "primary key" for storing and retrieving this information. + State string `json:"state"` + + // URL of the auth server (eg, PDS or entryway) + AuthServerURL string `json:"authserver_url"` + + // If the flow started with an account identifier (DID or handle), it should be persisted, to verify against the initial token response. + AccountDID *syntax.DID `json:"account_did,omitempty"` + + // OAuth scope strings + Scopes []string `json:"scopes"` + + // unique token in URI format, which will be used by the client in the auth flow redirect + RequestURI string `json:"request_uri"` + + // Full token endpoint URL + AuthServerTokenEndpoint string `json:"authserver_token_endpoint"` + + // Full revocation endpoint, if it exists + AuthServerRevocationEndpoint string `json:"authserver_revocation_endpoint,omitempty"` + + // The secret token/nonce which a code challenge was generated from + PKCEVerifier string `json:"pkce_verifier"` + + // Server-provided DPoP nonce from auth request (PAR) + DPoPAuthServerNonce string `json:"dpop_authserver_nonce"` + + // The secret cryptographic key generated by the client for this specific OAuth session + DPoPPrivateKeyMultibase string `json:"dpop_privatekey_multibase"` +} + +// The fields which are included in an initial token refresh request. These HTTP POST bodies are form-encoded, so use URL encoding syntax, not JSON. +type InitialTokenRequest struct { + // Client ID, aka client metadata URL + ClientID string `url:"client_id"` + + // Only used in initial token request. Auth server will validate that this matches the redirect URI used during the auth flow (resulting in the auth code) + RedirectURI string `url:"redirect_uri"` + + // Always `authorization_code` + GrantType string `url:"grant_type"` + + // Refresh token + RefreshToken string `url:"refresh_token"` + + // Authorization Code provided by the Auth Server via callback at the end of the auth request flow + Code string `url:"code"` + + // PKCE verifier string. Only included in initial token request + CodeVerifier string `url:"code_verifier"` + + // For confidential clients, must be "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ClientAssertionType *string `url:"client_assertion_type"` + + // For confidential clients, the signed client assertion JWT + ClientAssertion *string `url:"client_assertion"` +} + +// The fields which are included in a token refresh request. These HTTP POST bodies are form-encoded, so use URL encoding syntax, not JSON. +type RefreshTokenRequest struct { + // Client ID, aka client metadata URL + ClientID string `url:"client_id"` + + // Always `authorization_code` + GrantType string `url:"grant_type"` + + // Refresh token. + RefreshToken string `url:"refresh_token"` + + // For confidential clients, must be "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ClientAssertionType *string `url:"client_assertion_type"` + + // For confidential clients, the signed client assertion JWT + ClientAssertion *string `url:"client_assertion"` +} + +// Expected response from Auth Server token endpoint, both for initial token request and for refresh requests. +type TokenResponse struct { + Subject string `json:"sub"` + + // Usually expected to be the scopes that the client requested, but technically only a subset may have been approved, or additional scopes granted (?). + Scope string `json:"scope"` + + // Opaque access token, for requests to the resource server. + AccessToken string `json:"access_token"` + + // Refresh token, for doing additional token requests to the auth server. + RefreshToken string `json:"refresh_token"` +} + +// The fields which are included in a token revocation request. These HTTP POST bodies are form-encoded, so use URL encoding syntax, not JSON. +// +// Per https://datatracker.ietf.org/doc/html/rfc7009#section-2.1 +type RevocationRequest struct { + // Client ID, aka client metadata URL + ClientID string `url:"client_id"` + + // The token to revoke + Token string `url:"token"` + + // Either "access_token" or "refresh_token" + TokenTypeHint string `url:"token_type_hint"` + + // For confidential clients, must be "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ClientAssertionType *string `url:"client_assertion_type"` + + // For confidential clients, the signed client assertion JWT + ClientAssertion *string `url:"client_assertion"` +} + +// Returned by [ClientApp.ProcessCallback] if the AS signals an error in the redirect URL parameters, per rfc6749 section 4.1.2.1 +// +// NOTE: This is untrusted data and should not be e.g. rendered to HTML without appropriate escaping +type AuthRequestCallbackError struct { + ErrorCode string + ErrorDescription string + ErrorURI *syntax.URI +} + +func (e *AuthRequestCallbackError) Error() string { + res := "OAuth request callback error: " + e.ErrorCode + if e.ErrorDescription != "" { + res += ": " + e.ErrorDescription + } + if e.ErrorURI != nil { + res += " (" + e.ErrorURI.String() + ")" + } + return res +} diff --git a/atproto/auth/oauth/util.go b/atproto/auth/oauth/util.go new file mode 100644 index 000000000..b8ede04a8 --- /dev/null +++ b/atproto/auth/oauth/util.go @@ -0,0 +1,24 @@ +package oauth + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" +) + +// This is used both for PKCE challenges, and for pseudo-unique nonces to prevent token (JWT) replay. +func secureRandomBase64(sizeBytes uint) string { + buf := make([]byte, sizeBytes) + rand.Read(buf) + return base64.RawURLEncoding.EncodeToString(buf) +} + +// Computes an SHA-256 base64url-encoded challenge string, as used for PKCE. +func S256CodeChallenge(raw string) string { + b := sha256.Sum256([]byte(raw)) + return base64.RawURLEncoding.EncodeToString(b[:]) +} + +func strPtr(raw string) *string { + return &raw +} diff --git a/atproto/identity/apidir/apidir.go b/atproto/identity/apidir/apidir.go new file mode 100644 index 000000000..2840e7467 --- /dev/null +++ b/atproto/identity/apidir/apidir.go @@ -0,0 +1,226 @@ +package apidir + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/earthboundkid/versioninfo/v2" +) + +// Does HTTP requests to an identity server, using standard Lexicon endpoints +type APIDirectory struct { + Client *http.Client + // API service to make queries to. Includes schema, hostname, and port, but no path or trailing slash. Eg: "http://localhost:6600" + Host string + UserAgent string +} + +var _ identity.Directory = (*APIDirectory)(nil) +var _ identity.Resolver = (*APIDirectory)(nil) + +type identityBody struct { + DID syntax.DID `json:"did"` + Handle syntax.Handle `json:"handle"` + DIDDoc json.RawMessage `json:"didDoc"` +} + +type didBody struct { + DIDDoc json.RawMessage `json:"didDoc,omitempty"` +} + +type handleBody struct { + DID syntax.DID `json:"did"` +} + +func NewAPIDirectory(host string) APIDirectory { + return APIDirectory{ + Client: &http.Client{ + Timeout: time.Second * 10, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + IdleConnTimeout: time.Millisecond * 100, + MaxIdleConns: 100, + }, + }, + Host: host, + UserAgent: "indigo-apidir/" + versioninfo.Short(), + } +} + +// body: struct pointer which can be `json.Unmarshal()` +func (dir *APIDirectory) apiGet(ctx context.Context, u string, body any, errFail error, errNotFound error) error { + + req, err := http.NewRequestWithContext(ctx, "GET", u, nil) + if err != nil { + return fmt.Errorf("constructing HTTP request: %w", err) + } + if dir.UserAgent != "" { + req.Header.Set("User-Agent", dir.UserAgent) + } + resp, err := dir.Client.Do(req) + if err != nil { + return fmt.Errorf("%w: identity service HTTP: %w", errFail, err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("%w: identity service HTTP: %w", errFail, err) + } + + if resp.StatusCode == http.StatusNotFound { + return errNotFound + } + if resp.StatusCode != http.StatusOK { + // TODO: parse error body, handle more error conditions + return fmt.Errorf("%w: identity service HTTP: %d", errFail, resp.StatusCode) + } + + if err := json.Unmarshal(b, body); err != nil { + return fmt.Errorf("%w: identity service HTTP: %w", errFail, err) + } + return nil +} + +// body: struct pointer which can be `json.Unmarshal()` +func (dir *APIDirectory) apiPost(ctx context.Context, u string, reqBody []byte, body any, errFail error, errNotFound error) error { + req, err := http.NewRequestWithContext(ctx, "POST", u, bytes.NewBuffer(reqBody)) + if err != nil { + return fmt.Errorf("constructing HTTP request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + if dir.UserAgent != "" { + req.Header.Set("User-Agent", dir.UserAgent) + } + resp, err := dir.Client.Do(req) + if err != nil { + return fmt.Errorf("%w: identity service HTTP: %w", errFail, err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("%w: identity service HTTP: %w", errFail, err) + } + + if resp.StatusCode == http.StatusNotFound { + return errNotFound + } + if resp.StatusCode != http.StatusOK { + // TODO: parse error body, handle more error conditions + return fmt.Errorf("%w: identity service HTTP: %d", errFail, resp.StatusCode) + } + + if err := json.Unmarshal(b, body); err != nil { + return fmt.Errorf("%w: identity service HTTP: %w", errFail, err) + } + return nil +} + +func (dir *APIDirectory) ResolveDIDRaw(ctx context.Context, did syntax.DID) (json.RawMessage, error) { + var body didBody + u := dir.Host + "/xrpc/com.atproto.identity.resolveDid?did=" + did.String() + + start := time.Now() + err := dir.apiGet(ctx, u, &body, identity.ErrDIDResolutionFailed, identity.ErrDIDNotFound) + if err != nil { + didResolution.WithLabelValues("apidir", "error").Inc() + didResolutionDuration.WithLabelValues("apidir", "error").Observe(time.Since(start).Seconds()) + return nil, err + } + didResolution.WithLabelValues("apidir", "success").Inc() + didResolutionDuration.WithLabelValues("apidir", "success").Observe(time.Since(start).Seconds()) + + return body.DIDDoc, nil +} + +func (dir *APIDirectory) ResolveDID(ctx context.Context, did syntax.DID) (*identity.DIDDocument, error) { + raw, err := dir.ResolveDIDRaw(ctx, did) + if err != nil { + return nil, err + } + + var doc identity.DIDDocument + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("%w: JSON DID document parse: %w", identity.ErrDIDResolutionFailed, err) + } + return &doc, nil +} + +func (dir *APIDirectory) ResolveHandle(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { + handle = handle.Normalize() + var body handleBody + u := dir.Host + "/xrpc/com.atproto.identity.resolveHandle?handle=" + handle.String() + + start := time.Now() + err := dir.apiGet(ctx, u, &body, identity.ErrHandleResolutionFailed, identity.ErrHandleNotFound) + if err != nil { + handleResolution.WithLabelValues("apidir", "error").Inc() + handleResolutionDuration.WithLabelValues("apidir", "error").Observe(time.Since(start).Seconds()) + return "", err + } + handleResolution.WithLabelValues("apidir", "success").Inc() + handleResolutionDuration.WithLabelValues("apidir", "success").Observe(time.Since(start).Seconds()) + + return body.DID, nil +} + +func (dir *APIDirectory) Lookup(ctx context.Context, atid syntax.AtIdentifier) (*identity.Identity, error) { + var body identityBody + u := dir.Host + "/xrpc/com.atproto.identity.resolveIdentity?identifier=" + atid.String() + + // TODO: detect atid type, use that for errors? or just assume DID? + start := time.Now() + err := dir.apiGet(ctx, u, &body, identity.ErrDIDResolutionFailed, identity.ErrDIDNotFound) + if err != nil { + identityResolution.WithLabelValues("apidir", "error").Inc() + identityResolutionDuration.WithLabelValues("apidir", "error").Observe(time.Since(start).Seconds()) + return nil, err + } + identityResolution.WithLabelValues("apidir", "success").Inc() + identityResolutionDuration.WithLabelValues("apidir", "success").Observe(time.Since(start).Seconds()) + + var doc identity.DIDDocument + if err := json.Unmarshal(body.DIDDoc, &doc); err != nil { + return nil, fmt.Errorf("%w: JSON DID document parse: %w", identity.ErrDIDResolutionFailed, err) + } + + ident := identity.ParseIdentity(&doc) + ident.Handle = body.Handle + + return &ident, nil +} + +func (dir *APIDirectory) LookupHandle(ctx context.Context, handle syntax.Handle) (*identity.Identity, error) { + return dir.Lookup(ctx, handle.AtIdentifier()) +} + +func (dir *APIDirectory) LookupDID(ctx context.Context, did syntax.DID) (*identity.Identity, error) { + return dir.Lookup(ctx, did.AtIdentifier()) +} + +func (dir *APIDirectory) Purge(ctx context.Context, atid syntax.AtIdentifier) error { + + input := map[string]string{ + "identifier": atid.String(), + } + reqBody, err := json.Marshal(input) + if err != nil { + return err + } + + var body identityBody + u := dir.Host + "/xrpc/com.atproto.identity.refreshIdentity" + + if err := dir.apiPost(ctx, u, reqBody, &body, identity.ErrDIDResolutionFailed, identity.ErrDIDNotFound); err != nil { + return err + } + + return nil +} diff --git a/atproto/identity/apidir/doc.go b/atproto/identity/apidir/doc.go new file mode 100644 index 000000000..b9da52b2d --- /dev/null +++ b/atproto/identity/apidir/doc.go @@ -0,0 +1,12 @@ +/* +Identity Directory implementation which makes HTTP requests to a dedicated identity service. + +Implements both `identity.Directory` (Lookup methods) and `identity.Resolver` (Resolve methods). You may want to wrap this with a small in-process cache, eg `identity.CacheDirectory`. + +Makes use of standard Lexicons: + +- com.atproto.identity.resolveHandle +- com.atproto.identity.resolveDid +- com.atproto.identity.resolveIdentity +*/ +package apidir diff --git a/atproto/identity/apidir/examples_test.go b/atproto/identity/apidir/examples_test.go new file mode 100644 index 000000000..3296a7595 --- /dev/null +++ b/atproto/identity/apidir/examples_test.go @@ -0,0 +1,41 @@ +package apidir + +import ( + "context" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func ExampleAPIDirectory() { + // don't run this as a CI test! + //return + + ctx := context.Background() + + // will connect to the provided identity server (eg, a 'bluepages' instance) + dir := NewAPIDirectory("http://localhost:6600") + + handle, _ := syntax.ParseHandle("atproto.com") + did, _ := syntax.ParseDID("did:plc:ewvi7nxzyoun6zhxrhs64oiz") + + // low-level resolution of a handle (`identity.Resolver` interface) + atprotoDID, _ := dir.ResolveHandle(ctx, handle) + fmt.Println(atprotoDID) + + // low-level DID document resolution (`identity.Resolver` interface) + doc, err := dir.ResolveDID(ctx, did) + if err != nil { + panic(err) + } + fmt.Println(doc.Service) + + // higher-level identity resolution with accessors (`identity.Directory` interface) + ident, _ := dir.LookupHandle(ctx, handle) + fmt.Println(ident.PDSEndpoint()) + + /// Output: + // did:plc:ewvi7nxzyoun6zhxrhs64oiz + // [{#atproto_pds AtprotoPersonalDataServer https://enoki.us-east.host.bsky.network}] + // https://enoki.us-east.host.bsky.network +} diff --git a/atproto/identity/apidir/live_test.go b/atproto/identity/apidir/live_test.go new file mode 100644 index 000000000..b2b9722c0 --- /dev/null +++ b/atproto/identity/apidir/live_test.go @@ -0,0 +1,37 @@ +package apidir + +import ( + "context" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" +) + +func TestBasicLookups(t *testing.T) { + t.Skip("skipping live network test") + assert := assert.New(t) + ctx := context.Background() + var err error + + dir := NewAPIDirectory("http://localhost:6600") + + _, err = dir.LookupDID(ctx, syntax.DID("did:plc:ewvi7nxzyoun6zhxrhs64oiz")) + assert.NoError(err) + + _, err = dir.ResolveDID(ctx, syntax.DID("did:plc:ewvi7nxzyoun6zhxrhs64oiz")) + assert.NoError(err) + + _, err = dir.LookupHandle(ctx, syntax.Handle("atproto.com")) + assert.NoError(err) + + _, err = dir.ResolveHandle(ctx, syntax.Handle("atproto.com")) + assert.NoError(err) + + _, err = dir.LookupHandle(ctx, syntax.Handle("dummy-handle.atproto.com")) + assert.Error(err) + + _, err = dir.ResolveHandle(ctx, syntax.Handle("dummy-handle.atproto.com")) + assert.Error(err) +} diff --git a/atproto/identity/apidir/metrics.go b/atproto/identity/apidir/metrics.go new file mode 100644 index 000000000..f54294c57 --- /dev/null +++ b/atproto/identity/apidir/metrics.go @@ -0,0 +1,39 @@ +package apidir + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var handleResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_apidir_resolve_handle", + Help: "ATProto handle resolutions", +}, []string{"directory", "status"}) + +var handleResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_apidir_resolve_handle_duration", + Help: "Time to resolve a handle", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) + +var didResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_apidir_resolve_did", + Help: "ATProto DID resolutions", +}, []string{"directory", "status"}) + +var didResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_apidir_resolve_did_duration", + Help: "Time to resolve a DID", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) + +var identityResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_apidir_resolve_identity", + Help: "ATProto combined identity resolutions", +}, []string{"directory", "status"}) + +var identityResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_apidir_resolve_identity_duration", + Help: "Time to resolve a combined identity", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) diff --git a/atproto/identity/base_directory.go b/atproto/identity/base_directory.go new file mode 100644 index 000000000..6dbf2e19b --- /dev/null +++ b/atproto/identity/base_directory.go @@ -0,0 +1,116 @@ +package identity + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + + "github.com/bluesky-social/indigo/atproto/syntax" + "golang.org/x/time/rate" +) + +// The zero value ('BaseDirectory{}') is a usable Directory. +type BaseDirectory struct { + // if non-empty, this string should have URL method, hostname, and optional port; it should not have a path or trailing slash + PLCURL string + // If not nil, this limiter will be used to rate-limit requests to the PLCURL + PLCLimiter *rate.Limiter + // If not nil, this function will be called inline with DID Web lookups, and can be used to limit the number of requests to a given hostname + DIDWebLimitFunc func(ctx context.Context, hostname string) error + // HTTP client used for did:web, did:plc, and HTTP (well-known) handle resolution + HTTPClient http.Client + // DNS resolver used for DNS handle resolution. Calling code can use a custom Dialer to query against a specific DNS server, or re-implement the interface for even more control over the resolution process + Resolver net.Resolver + // when doing DNS handle resolution, should this resolver attempt re-try against an authoritative nameserver if the first TXT lookup fails? + TryAuthoritativeDNS bool + // set of handle domain suffixes for for which DNS handle resolution will be skipped + SkipDNSDomainSuffixes []string + // set of fallback DNS servers (eg, domain registrars) to try as a fallback. each entry should be "ip:port", eg "8.8.8.8:53" + FallbackDNSServers []string + // skips bi-directional verification of handles when doing DID lookups (eg, `LookupDID`). Does not impact direct resolution (`ResolveHandle`) or handle-specific lookup (`LookupHandle`). + // + // The intended use-case for this flag is as an optimization for services which do not care about handles, but still want to use the `Directory` interface (instead of `ResolveDID`). For example, relay implementations, or services validating inter-service auth requests. + SkipHandleVerification bool + // User-Agent header for HTTP requests. Optional (ignored if empty string). + UserAgent string +} + +var _ Directory = (*BaseDirectory)(nil) +var _ Resolver = (*BaseDirectory)(nil) + +func (d *BaseDirectory) LookupHandle(ctx context.Context, h syntax.Handle) (*Identity, error) { + h = h.Normalize() + did, err := d.ResolveHandle(ctx, h) + if err != nil { + return nil, err + } + doc, err := d.ResolveDID(ctx, did) + if err != nil { + return nil, err + } + ident := ParseIdentity(doc) + declared, err := ident.DeclaredHandle() + if err != nil { + return nil, fmt.Errorf("could not verify handle/DID match: %w", err) + } + // NOTE: DeclaredHandle() returns a normalized handle, and we already normalized 'h' above + if declared != h { + return nil, fmt.Errorf("%w: %s != %s", ErrHandleMismatch, declared, h) + } + ident.Handle = declared + + return &ident, nil +} + +func (d *BaseDirectory) LookupDID(ctx context.Context, did syntax.DID) (*Identity, error) { + doc, err := d.ResolveDID(ctx, did) + if err != nil { + return nil, err + } + ident := ParseIdentity(doc) + if d.SkipHandleVerification { + ident.Handle = syntax.HandleInvalid + return &ident, nil + } + declared, err := ident.DeclaredHandle() + if errors.Is(err, ErrHandleNotDeclared) { + ident.Handle = syntax.HandleInvalid + } else if err != nil { + return nil, fmt.Errorf("could not parse handle from DID document: %w", err) + } else { + // if a handle was declared, resolve it + resolvedDID, err := d.ResolveHandle(ctx, declared) + if err != nil { + if errors.Is(err, ErrHandleNotFound) || errors.Is(err, ErrHandleResolutionFailed) { + ident.Handle = syntax.HandleInvalid + } else { + return nil, err + } + } else if resolvedDID != did { + ident.Handle = syntax.HandleInvalid + } else { + ident.Handle = declared + } + } + + return &ident, nil +} + +func (d *BaseDirectory) Lookup(ctx context.Context, a syntax.AtIdentifier) (*Identity, error) { + handle, err := a.AsHandle() + if nil == err { // if *not* an error + return d.LookupHandle(ctx, handle) + } + did, err := a.AsDID() + if nil == err { // if *not* an error + return d.LookupDID(ctx, did) + } + return nil, fmt.Errorf("at-identifier neither a Handle nor a DID") +} + +func (d *BaseDirectory) Purge(ctx context.Context, atid syntax.AtIdentifier) error { + // BaseDirectory itself does not implement caching + return nil +} diff --git a/atproto/identity/cache_directory.go b/atproto/identity/cache_directory.go new file mode 100644 index 000000000..de4dad7da --- /dev/null +++ b/atproto/identity/cache_directory.go @@ -0,0 +1,287 @@ +package identity + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/hashicorp/golang-lru/v2/expirable" +) + +// CacheDirectory is an implementation of identity.Directory with local cache of Handle and DID +type CacheDirectory struct { + Inner Directory + ErrTTL time.Duration + InvalidHandleTTL time.Duration + handleCache *expirable.LRU[syntax.Handle, handleEntry] + identityCache *expirable.LRU[syntax.DID, identityEntry] + didLookupChans sync.Map + handleLookupChans sync.Map +} + +type handleEntry struct { + Updated time.Time + DID syntax.DID + Err error +} + +type identityEntry struct { + Updated time.Time + Identity *Identity + Err error +} + +var _ Directory = (*CacheDirectory)(nil) + +// Capacity of zero means unlimited size. Similarly, ttl of zero means unlimited duration. +func NewCacheDirectory(inner Directory, capacity int, hitTTL, errTTL, invalidHandleTTL time.Duration) CacheDirectory { + return CacheDirectory{ + ErrTTL: errTTL, + InvalidHandleTTL: invalidHandleTTL, + Inner: inner, + handleCache: expirable.NewLRU[syntax.Handle, handleEntry](capacity, nil, hitTTL), + identityCache: expirable.NewLRU[syntax.DID, identityEntry](capacity, nil, hitTTL), + } +} + +func (d *CacheDirectory) isHandleStale(e *handleEntry) bool { + if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { + return true + } + return false +} + +func (d *CacheDirectory) isIdentityStale(e *identityEntry) bool { + if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { + return true + } + if e.Identity != nil && e.Identity.Handle.IsInvalidHandle() && time.Since(e.Updated) > d.InvalidHandleTTL { + return true + } + return false +} + +func (d *CacheDirectory) updateHandle(ctx context.Context, h syntax.Handle) handleEntry { + ident, err := d.Inner.LookupHandle(ctx, h) + if err != nil { + he := handleEntry{ + Updated: time.Now(), + DID: "", + Err: err, + } + d.handleCache.Add(h, he) + return he + } + + entry := identityEntry{ + Updated: time.Now(), + Identity: ident, + Err: nil, + } + he := handleEntry{ + Updated: time.Now(), + DID: ident.DID, + Err: nil, + } + + d.identityCache.Add(ident.DID, entry) + d.handleCache.Add(ident.Handle, he) + return he +} + +func (d *CacheDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { + h = h.Normalize() + if h.IsInvalidHandle() { + return "", fmt.Errorf("can not resolve handle: %w", ErrInvalidHandle) + } + start := time.Now() + entry, ok := d.handleCache.Get(h) + if ok && !d.isHandleStale(&entry) { + handleCacheHits.Inc() + handleResolution.WithLabelValues("lru", "cached").Inc() + handleResolutionDuration.WithLabelValues("lru", "cached").Observe(time.Since(start).Seconds()) + return entry.DID, entry.Err + } + handleCacheMisses.Inc() + + // Coalesce multiple requests for the same Handle + res := make(chan struct{}) + val, loaded := d.handleLookupChans.LoadOrStore(h.String(), res) + if loaded { + handleRequestsCoalesced.Inc() + handleResolution.WithLabelValues("lru", "coalesced").Inc() + handleResolutionDuration.WithLabelValues("lru", "coalesced").Observe(time.Since(start).Seconds()) + // Wait for the result from the pending request + select { + case <-val.(chan struct{}): + // The result should now be in the cache + entry, ok := d.handleCache.Get(h) + if ok && !d.isHandleStale(&entry) { + return entry.DID, entry.Err + } + return "", fmt.Errorf("identity not found in cache after coalesce returned") + case <-ctx.Done(): + return "", ctx.Err() + } + } + + // Update the Handle Entry from PLC and cache the result + newEntry := d.updateHandle(ctx, h) + + // Cleanup the coalesce map and close the results channel + d.handleLookupChans.Delete(h.String()) + // Callers waiting will now get the result from the cache + close(res) + + if newEntry.Err != nil { + handleResolution.WithLabelValues("lru", "error").Inc() + handleResolutionDuration.WithLabelValues("lru", "error").Observe(time.Since(start).Seconds()) + return "", newEntry.Err + } + if newEntry.DID != "" { + handleResolution.WithLabelValues("lru", "success").Inc() + handleResolutionDuration.WithLabelValues("lru", "success").Observe(time.Since(start).Seconds()) + return newEntry.DID, nil + } + return "", fmt.Errorf("unexpected control-flow error") +} + +func (d *CacheDirectory) updateDID(ctx context.Context, did syntax.DID) identityEntry { + ident, err := d.Inner.LookupDID(ctx, did) + // persist the identity lookup error, instead of processing it immediately + entry := identityEntry{ + Updated: time.Now(), + Identity: ident, + Err: err, + } + var he *handleEntry + // if *not* an error, then also update the handle cache + if nil == err && !ident.Handle.IsInvalidHandle() { + he = &handleEntry{ + Updated: time.Now(), + DID: did, + Err: nil, + } + } + + d.identityCache.Add(did, entry) + if he != nil { + d.handleCache.Add(ident.Handle, *he) + } + return entry +} + +func (d *CacheDirectory) LookupDID(ctx context.Context, did syntax.DID) (*Identity, error) { + id, _, err := d.LookupDIDWithCacheState(ctx, did) + return id, err +} + +func (d *CacheDirectory) LookupDIDWithCacheState(ctx context.Context, did syntax.DID) (*Identity, bool, error) { + start := time.Now() + entry, ok := d.identityCache.Get(did) + if ok && !d.isIdentityStale(&entry) { + identityCacheHits.Inc() + didResolution.WithLabelValues("lru", "cached").Inc() + didResolutionDuration.WithLabelValues("lru", "cached").Observe(time.Since(start).Seconds()) + return entry.Identity, true, entry.Err + } + identityCacheMisses.Inc() + + // Coalesce multiple requests for the same DID + res := make(chan struct{}) + val, loaded := d.didLookupChans.LoadOrStore(did.String(), res) + if loaded { + identityRequestsCoalesced.Inc() + didResolution.WithLabelValues("lru", "coalesced").Inc() + didResolutionDuration.WithLabelValues("lru", "coalesced").Observe(time.Since(start).Seconds()) + // Wait for the result from the pending request + select { + case <-val.(chan struct{}): + // The result should now be in the cache + entry, ok := d.identityCache.Get(did) + if ok && !d.isIdentityStale(&entry) { + return entry.Identity, false, entry.Err + } + return nil, false, fmt.Errorf("identity not found in cache after coalesce returned") + case <-ctx.Done(): + return nil, false, ctx.Err() + } + } + + // Update the Identity Entry from PLC and cache the result + newEntry := d.updateDID(ctx, did) + + // Cleanup the coalesce map and close the results channel + d.didLookupChans.Delete(did.String()) + // Callers waiting will now get the result from the cache + close(res) + + if newEntry.Err != nil { + didResolution.WithLabelValues("lru", "error").Inc() + didResolutionDuration.WithLabelValues("lru", "error").Observe(time.Since(start).Seconds()) + return nil, false, newEntry.Err + } + if newEntry.Identity != nil { + didResolution.WithLabelValues("lru", "success").Inc() + didResolutionDuration.WithLabelValues("lru", "success").Observe(time.Since(start).Seconds()) + return newEntry.Identity, false, nil + } + return nil, false, fmt.Errorf("unexpected control-flow error") +} + +func (d *CacheDirectory) LookupHandle(ctx context.Context, h syntax.Handle) (*Identity, error) { + ident, _, err := d.LookupHandleWithCacheState(ctx, h) + return ident, err +} + +func (d *CacheDirectory) LookupHandleWithCacheState(ctx context.Context, h syntax.Handle) (*Identity, bool, error) { + h = h.Normalize() + did, err := d.ResolveHandle(ctx, h) + if err != nil { + return nil, false, err + } + ident, hit, err := d.LookupDIDWithCacheState(ctx, did) + if err != nil { + return nil, hit, err + } + + declared, err := ident.DeclaredHandle() + if err != nil { + return nil, hit, fmt.Errorf("could not verify handle/DID mapping: %w", err) + } + // NOTE: DeclaredHandle() returns a normalized handle, and we already normalized 'h' above + if declared != h { + return nil, hit, fmt.Errorf("%w: %s != %s", ErrHandleMismatch, declared, h) + } + return ident, hit, nil +} + +func (d *CacheDirectory) Lookup(ctx context.Context, a syntax.AtIdentifier) (*Identity, error) { + handle, err := a.AsHandle() + if nil == err { // if not an error, is a handle + return d.LookupHandle(ctx, handle) + } + did, err := a.AsDID() + if nil == err { // if not an error, is a DID + return d.LookupDID(ctx, did) + } + return nil, fmt.Errorf("at-identifier neither a Handle nor a DID") +} + +func (d *CacheDirectory) Purge(ctx context.Context, atid syntax.AtIdentifier) error { + handle, err := atid.AsHandle() + if nil == err { // if not an error, is a handle + handle = handle.Normalize() + d.handleCache.Remove(handle) + return nil + } + did, err := atid.AsDID() + if nil == err { // if not an error, is a DID + d.identityCache.Remove(did) + return nil + } + return fmt.Errorf("at-identifier neither a Handle nor a DID") +} diff --git a/atproto/identity/cmd/atp-id/main.go b/atproto/identity/cmd/atp-id/main.go new file mode 100644 index 000000000..aebb17084 --- /dev/null +++ b/atproto/identity/cmd/atp-id/main.go @@ -0,0 +1,116 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "atp-id", + Usage: "informal debugging CLI tool for atproto identities", + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "lookup", + Usage: "fully resolve an at-identifier (DID or handle)", + ArgsUsage: "", + Action: runLookup, + }, + &cli.Command{ + Name: "resolve-handle", + Usage: "resolve a handle to DID", + ArgsUsage: "", + Action: runResolveHandle, + }, + &cli.Command{ + Name: "resolve-did", + Usage: "resolve a DID to DID Document", + ArgsUsage: "", + Action: runResolveDID, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + if err := app.Run(context.Background(), os.Args); err != nil { + slog.Error("command failed", "error", err) + os.Exit(-1) + } +} + +func runLookup(ctx context.Context, cmd *cli.Command) error { + s := cmd.Args().First() + if s == "" { + return fmt.Errorf("need to provide identifier as an argument") + } + + id, err := syntax.ParseAtIdentifier(s) + if err != nil { + return err + } + slog.Info("valid syntax", "at-identifier", id) + + dir := identity.DefaultDirectory() + acc, err := dir.Lookup(ctx, id) + if err != nil { + return err + } + fmt.Println(acc) + return nil +} + +func runResolveHandle(ctx context.Context, cmd *cli.Command) error { + s := cmd.Args().First() + if s == "" { + return fmt.Errorf("need to provide handle as an argument") + } + + handle, err := syntax.ParseHandle(s) + if err != nil { + return err + } + slog.Info("valid syntax", "handle", handle) + + d := identity.BaseDirectory{} + did, err := d.ResolveHandle(ctx, handle) + if err != nil { + return err + } + fmt.Println(did) + return nil +} + +func runResolveDID(ctx context.Context, cmd *cli.Command) error { + s := cmd.Args().First() + if s == "" { + fmt.Println("need to provide DID as an argument") + os.Exit(-1) + } + + did, err := syntax.ParseDID(s) + if err != nil { + fmt.Println(err) + os.Exit(-1) + } + slog.Info("valid syntax", "did", did) + + d := identity.BaseDirectory{} + doc, err := d.ResolveDID(ctx, did) + if err != nil { + return err + } + jsonBytes, err := json.Marshal(&doc) + if err != nil { + return err + } + fmt.Println(string(jsonBytes)) + return nil +} diff --git a/atproto/identity/did.go b/atproto/identity/did.go new file mode 100644 index 000000000..982fd4925 --- /dev/null +++ b/atproto/identity/did.go @@ -0,0 +1,169 @@ +package identity + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Resolves a DID to a parsed `DIDDocument` struct. +// +// This method does not bi-directionally verify handles. Most atproto-specific code should use the `identity.Directory` interface ("Lookup" methods), which implement that check by default, and provide more ergonomic helpers for working with atproto-relevant information in DID documents. +// +// Note that the `DIDDocument` might not include all the information in the original document. Use `ResolveDIDRaw()` to get the full original JSON. +func (d *BaseDirectory) ResolveDID(ctx context.Context, did syntax.DID) (*DIDDocument, error) { + b, err := d.resolveDIDBytes(ctx, did) + if err != nil { + return nil, err + } + + var doc DIDDocument + if err := json.Unmarshal(b, &doc); err != nil { + return nil, fmt.Errorf("%w: JSON DID document parse: %w", ErrDIDResolutionFailed, err) + } + if doc.DID != did { + return nil, fmt.Errorf("document ID did not match DID") + } + return &doc, nil +} + +// Low-level method for resolving a DID to a raw JSON document. +// +// This method does not parse the DID document into an atproto-specific format, and does not bi-directionally verify handles. Most atproto-specific code should use the "Lookup*" methods, which do implement that functionality. +func (d *BaseDirectory) ResolveDIDRaw(ctx context.Context, did syntax.DID) (json.RawMessage, error) { + b, err := d.resolveDIDBytes(ctx, did) + if err != nil { + return nil, err + } + + // parse as doc, to validate at least some syntax + var doc DIDDocument + if err := json.Unmarshal(b, &doc); err != nil { + return nil, fmt.Errorf("%w: JSON DID document parse: %w", ErrDIDResolutionFailed, err) + } + if doc.DID != did { + return nil, fmt.Errorf("document ID did not match DID") + } + + return json.RawMessage(b), nil +} + +func (d *BaseDirectory) resolveDIDBytes(ctx context.Context, did syntax.DID) ([]byte, error) { + var b []byte + var err error + start := time.Now() + switch did.Method() { + case "web": + b, err = d.resolveDIDWeb(ctx, did) + case "plc": + b, err = d.resolveDIDPLC(ctx, did) + default: + return nil, fmt.Errorf("DID method not supported: %s", did.Method()) + } + elapsed := time.Since(start) + slog.Debug("resolve DID", "did", did, "err", err, "duration_ms", elapsed.Milliseconds()) + return b, err +} + +func (d *BaseDirectory) resolveDIDWeb(ctx context.Context, did syntax.DID) ([]byte, error) { + if did.Method() != "web" { + return nil, fmt.Errorf("expected a did:web, got: %s", did) + } + hostname := did.Identifier() + handle, err := syntax.ParseHandle(hostname) + if err != nil { + return nil, fmt.Errorf("did:web identifier not a simple hostname: %s", hostname) + } + if !handle.AllowedTLD() { + return nil, fmt.Errorf("did:web hostname has disallowed TLD: %s", hostname) + } + + // TODO: allow ctx to specify unsafe http:// resolution, for testing? + + if d.DIDWebLimitFunc != nil { + if err := d.DIDWebLimitFunc(ctx, hostname); err != nil { + return nil, fmt.Errorf("did:web limit func returned an error for (%s): %w", hostname, err) + } + } + + req, err := http.NewRequestWithContext(ctx, "GET", "https://"+hostname+"/.well-known/did.json", nil) + if err != nil { + return nil, fmt.Errorf("constructing HTTP request for did:web resolution: %w", err) + } + if d.UserAgent != "" { + req.Header.Set("User-Agent", d.UserAgent) + } + + resp, err := d.HTTPClient.Do(req) + + // look for NXDOMAIN + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsNotFound { + return nil, fmt.Errorf("%w: DNS NXDOMAIN", ErrDIDNotFound) + } + } + if err != nil { + return nil, fmt.Errorf("%w: did:web HTTP well-known fetch: %w", ErrDIDResolutionFailed, err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("%w: did:web HTTP status 404", ErrDIDNotFound) + } + if resp.StatusCode != http.StatusOK { + io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("%w: did:web HTTP status %d", ErrDIDResolutionFailed, resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} + +func (d *BaseDirectory) resolveDIDPLC(ctx context.Context, did syntax.DID) ([]byte, error) { + if did.Method() != "plc" { + return nil, fmt.Errorf("expected a did:plc, got: %s", did) + } + + plcURL := d.PLCURL + if plcURL == "" { + plcURL = DefaultPLCURL + } + + if d.PLCLimiter != nil { + if err := d.PLCLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("failed to wait for PLC limiter: %w", err) + } + } + + req, err := http.NewRequestWithContext(ctx, "GET", plcURL+"/"+did.String(), nil) + if err != nil { + return nil, fmt.Errorf("constructing HTTP request for did:plc resolution: %w", err) + } + if d.UserAgent != "" { + req.Header.Set("User-Agent", d.UserAgent) + } + + resp, err := d.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: PLC directory lookup: %w", ErrDIDResolutionFailed, err) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("%w: PLC directory 404", ErrDIDNotFound) + } + if resp.StatusCode != http.StatusOK { + io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("%w: PLC directory status %d", ErrDIDResolutionFailed, resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} diff --git a/atproto/identity/diddoc.go b/atproto/identity/diddoc.go new file mode 100644 index 000000000..4c7070c20 --- /dev/null +++ b/atproto/identity/diddoc.go @@ -0,0 +1,25 @@ +package identity + +import ( + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type DIDDocument struct { + DID syntax.DID `json:"id"` + AlsoKnownAs []string `json:"alsoKnownAs,omitempty"` + VerificationMethod []DocVerificationMethod `json:"verificationMethod,omitempty"` + Service []DocService `json:"service,omitempty"` +} + +type DocVerificationMethod struct { + ID string `json:"id"` + Type string `json:"type"` + Controller string `json:"controller"` + PublicKeyMultibase string `json:"publicKeyMultibase"` +} + +type DocService struct { + ID string `json:"id"` + Type string `json:"type"` + ServiceEndpoint string `json:"serviceEndpoint"` +} diff --git a/atproto/identity/diddoc_test.go b/atproto/identity/diddoc_test.go new file mode 100644 index 000000000..e30e074a4 --- /dev/null +++ b/atproto/identity/diddoc_test.go @@ -0,0 +1,85 @@ +package identity + +import ( + "encoding/json" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDIDDocParse(t *testing.T) { + assert := assert.New(t) + docFiles := []string{ + "testdata/did_plc_doc.json", + "testdata/did_plc_doc_legacy.json", + } + for _, path := range docFiles { + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + docBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var doc DIDDocument + err = json.Unmarshal(docBytes, &doc) + assert.NoError(err) + + id := ParseIdentity(&doc) + + assert.Equal("did:plc:ewvi7nxzyoun6zhxrhs64oiz", id.DID.String()) + assert.Equal([]string{"at://atproto.com"}, id.AlsoKnownAs) + pk, err := id.PublicKey() + assert.NoError(err) + assert.NotNil(pk) + assert.Equal("https://bsky.social", id.PDSEndpoint()) + hdl, err := id.DeclaredHandle() + assert.NoError(err) + assert.Equal("atproto.com", hdl.String()) + + // NOTE: doesn't work if 'id' was in long form + if path != "testdata/did_plc_doc_legacy.json" { + assert.Equal(doc, id.DIDDocument()) + } + } +} + +func TestDIDDocFeedGenParse(t *testing.T) { + assert := assert.New(t) + f, err := os.Open("testdata/did_web_doc.json") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + docBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var doc DIDDocument + err = json.Unmarshal(docBytes, &doc) + assert.NoError(err) + + id := ParseIdentity(&doc) + + assert.Equal("did:web:discover.bsky.social", id.DID.String()) + assert.Equal([]string{}, id.AlsoKnownAs) + pk, err := id.PublicKey() + assert.Error(err) + assert.ErrorIs(err, ErrKeyNotDeclared) + assert.Nil(pk) + assert.Equal("", id.PDSEndpoint()) + hdl, err := id.DeclaredHandle() + assert.Error(err) + assert.Empty(hdl) + svc, ok := id.Services["bsky_fg"] + assert.True(ok) + assert.Equal("https://discover.bsky.social", svc.URL) +} diff --git a/atproto/identity/directory.go b/atproto/identity/directory.go new file mode 100644 index 000000000..c67d16a2e --- /dev/null +++ b/atproto/identity/directory.go @@ -0,0 +1,90 @@ +package identity + +import ( + "context" + "errors" + "net" + "net/http" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/earthboundkid/versioninfo/v2" +) + +// Ergonomic interface for atproto identity lookup, by DID or handle. +// +// The "Lookup" methods resolve identities (handle and DID), and return results in a compact, opinionated struct (`Identity`). They do bi-directional handle/DID verification by default. Clients and services should use these methods by default, instead of resolving handles or DIDs separately. +// +// Looking up a handle which fails to resolve, or don't match DID alsoKnownAs, returns an error. When looking up a DID, if the handle does not resolve back to the DID, the lookup succeeds and returns an `Identity` where the Handle is the special `handle.invalid` value. +// +// Some example implementations of this interface could be: +// - basic direct resolution on every call +// - local in-memory caching layer to reduce network hits +// - API client, which just makes requests to PDS (or other remote service) +// - client for shared network cache (eg, Redis) +type Directory interface { + LookupHandle(ctx context.Context, handle syntax.Handle) (*Identity, error) + LookupDID(ctx context.Context, did syntax.DID) (*Identity, error) + Lookup(ctx context.Context, atid syntax.AtIdentifier) (*Identity, error) + + // Flushes any cache of the indicated identifier. If directory is not using caching, can ignore this. + Purge(ctx context.Context, atid syntax.AtIdentifier) error +} + +// Indicates that handle resolution failed. A wrapped error may provide more context. This is only returned when looking up a handle, not when looking up a DID. +var ErrHandleResolutionFailed = errors.New("handle resolution failed") + +// Indicates that resolution process completed successfully, but handle does not exist. This is only returned when looking up a handle, not when looking up a DID. +var ErrHandleNotFound = errors.New("handle not found") + +// Indicates that resolution process completed successfully, handle mapped to a different DID. This is only returned when looking up a handle, not when looking up a DID. +var ErrHandleMismatch = errors.New("handle/DID mismatch") + +// Indicates that DID document did not include any handle ("alsoKnownAs"). This is only returned when looking up a handle, not when looking up a DID. +var ErrHandleNotDeclared = errors.New("DID document did not declare a handle") + +// Handle top-level domain (TLD) is one of the special "Reserved" suffixes, and not allowed for atproto use +var ErrHandleReservedTLD = errors.New("handle top-level domain is disallowed") + +// Indicates that resolution process completed successfully, but the DID does not exist. +var ErrDIDNotFound = errors.New("DID not found") + +// Indicates that DID resolution process failed. A wrapped error may provide more context. +var ErrDIDResolutionFailed = errors.New("DID resolution failed") + +// Indicates that DID document did not include a public key with the specified ID +var ErrKeyNotDeclared = errors.New("DID document did not declare a relevant public key") + +// Handle was invalid, in a situation where a valid handle is required. +var ErrInvalidHandle = errors.New("invalid handle") + +var DefaultPLCURL = "https://plc.directory" + +// Returns a reasonable Directory implementation for applications +func DefaultDirectory() Directory { + base := BaseDirectory{ + PLCURL: DefaultPLCURL, + HTTPClient: http.Client{ + Timeout: time.Second * 10, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + // would want this around 100ms for services doing lots of handle resolution. Impacts PLC connections as well, but not too bad. + IdleConnTimeout: time.Millisecond * 1000, + MaxIdleConns: 100, + }, + }, + Resolver: net.Resolver{ + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{Timeout: time.Second * 3} + return d.DialContext(ctx, network, address) + }, + }, + TryAuthoritativeDNS: true, + // primary Bluesky PDS instance only supports HTTP resolution method + SkipDNSDomainSuffixes: []string{".bsky.social"}, + UserAgent: "indigo-identity/" + versioninfo.Short(), + } + cached := NewCacheDirectory(&base, 250_000, time.Hour*24, time.Minute*2, time.Minute*5) + return &cached +} diff --git a/atproto/identity/doc.go b/atproto/identity/doc.go new file mode 100644 index 000000000..32bef3313 --- /dev/null +++ b/atproto/identity/doc.go @@ -0,0 +1,6 @@ +/* +Package identity provides types and routines for resolving handles and DIDs from the network + +The two main abstractions are a Directory interface for identity service implementations, and an Identity struct which represents core identity information relevant to atproto. The Directory interface can be nested, somewhat like HTTP middleware, to provide caching, observability, or other bespoke needs in more complex systems. +*/ +package identity diff --git a/atproto/identity/handle.go b/atproto/identity/handle.go new file mode 100644 index 000000000..51220ce58 --- /dev/null +++ b/atproto/identity/handle.go @@ -0,0 +1,238 @@ +package identity + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "strings" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func parseTXTResp(res []string) (syntax.DID, error) { + for _, s := range res { + if strings.HasPrefix(s, "did=") { + parts := strings.SplitN(s, "=", 2) + did, err := syntax.ParseDID(parts[1]) + if err != nil { + return "", fmt.Errorf("%w: invalid DID in handle DNS record: %w", ErrHandleResolutionFailed, err) + } + return did, nil + } + } + return "", ErrHandleNotFound +} + +// Does not cross-verify, only does the handle resolution step. +func (d *BaseDirectory) ResolveHandleDNS(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { + res, err := d.Resolver.LookupTXT(ctx, "_atproto."+handle.String()) + // check for NXDOMAIN + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsNotFound { + return "", fmt.Errorf("%w: %s", ErrHandleNotFound, handle) + } + } + if err != nil { + return "", fmt.Errorf("%w: %w", ErrHandleResolutionFailed, err) + } + return parseTXTResp(res) +} + +// this is a variant of ResolveHandleDNS which first does an authoritative nameserver lookup, then queries there +func (d *BaseDirectory) ResolveHandleDNSAuthoritative(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { + // lookup nameserver using configured resolver + resNS, err := d.Resolver.LookupNS(ctx, handle.String()) + // check for NXDOMAIN + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsNotFound { + return "", ErrHandleNotFound + } + } + if err != nil { + return "", fmt.Errorf("%w: DNS error: %w", ErrHandleResolutionFailed, err) + } + if len(resNS) == 0 { + return "", ErrHandleNotFound + } + ns := resNS[0].Host + if !strings.Contains(ns, ":") { + ns = ns + ":53" + } + + // create a custom resolver to use the specific nameserver for TXT lookup + resolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + rd := net.Dialer{ + Timeout: time.Second * 5, + } + return rd.DialContext(ctx, network, ns) + }, + } + res, err := resolver.LookupTXT(ctx, "_atproto."+handle.String()) + // check for NXDOMAIN + if errors.As(err, &dnsErr) { + if dnsErr.IsNotFound { + return "", ErrHandleNotFound + } + } + if err != nil { + return "", fmt.Errorf("%w: DNS resolution failed: %w", ErrHandleResolutionFailed, err) + } + return parseTXTResp(res) +} + +// variant of ResolveHandleDNS which uses any configured fallback DNS servers +func (d *BaseDirectory) ResolveHandleDNSFallback(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { + retErr := fmt.Errorf("no fallback servers configured") + var dnsErr *net.DNSError + for _, ns := range d.FallbackDNSServers { + // create a custom resolver to use the specific nameserver for TXT lookup + resolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + rd := net.Dialer{ + Timeout: time.Second * 5, + } + return rd.DialContext(ctx, network, ns) + }, + } + res, err := resolver.LookupTXT(ctx, "_atproto."+handle.String()) + // check for NXDOMAIN + if errors.As(err, &dnsErr) { + if dnsErr.IsNotFound { + retErr = ErrHandleNotFound + continue + } + } + if err != nil { + retErr = fmt.Errorf("%w: %w", ErrHandleResolutionFailed, err) + continue + } + ret, err := parseTXTResp(res) + if err != nil { + retErr = err + continue + } + return ret, nil + } + return "", retErr +} + +func (d *BaseDirectory) ResolveHandleWellKnown(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/.well-known/atproto-did", handle), nil) + if err != nil { + return "", fmt.Errorf("constructing HTTP request for handle resolution: %w", err) + } + if d.UserAgent != "" { + req.Header.Set("User-Agent", d.UserAgent) + } + + resp, err := d.HTTPClient.Do(req) + if err != nil { + // check for NXDOMAIN + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsNotFound { + return "", fmt.Errorf("%w: DNS NXDOMAIN for HTTP well-known resolution of %s", ErrHandleNotFound, handle) + } + } + return "", fmt.Errorf("%w: HTTP well-known request error: %w", ErrHandleResolutionFailed, err) + } + defer resp.Body.Close() + if resp.ContentLength > 2048 { + // NOTE: intentionally not draining body + return "", fmt.Errorf("%w: HTTP well-known body too large for %s", ErrHandleResolutionFailed, handle) + } + if resp.StatusCode == http.StatusNotFound { + io.Copy(io.Discard, resp.Body) + return "", fmt.Errorf("%w: HTTP 404 for %s", ErrHandleNotFound, handle) + } + if resp.StatusCode != http.StatusOK { + io.Copy(io.Discard, resp.Body) + return "", fmt.Errorf("%w: HTTP well-known status %d for %s", ErrHandleResolutionFailed, resp.StatusCode, handle) + } + + b, err := io.ReadAll(io.LimitReader(resp.Body, 2048)) + if err != nil { + return "", fmt.Errorf("%w: HTTP well-known body read for %s: %w", ErrHandleResolutionFailed, handle, err) + } + line := strings.TrimSpace(string(b)) + outDid, err := syntax.ParseDID(line) + if err != nil { + return outDid, fmt.Errorf("%w: invalid DID in HTTP well-known for %s", ErrHandleResolutionFailed, handle) + } + return outDid, err +} + +func (d *BaseDirectory) ResolveHandle(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { + // TODO: *could* do resolution in parallel, but expecting that sequential is sufficient to start + var dnsErr error + var did syntax.DID + + handle = handle.Normalize() + + if handle.IsInvalidHandle() { + return "", fmt.Errorf("can not resolve handle: %w", ErrInvalidHandle) + } + + if !handle.AllowedTLD() { + return "", ErrHandleReservedTLD + } + + tryDNS := true + for _, suffix := range d.SkipDNSDomainSuffixes { + if strings.HasSuffix(handle.String(), suffix) { + tryDNS = false + break + } + } + + if tryDNS { + start := time.Now() + triedAuthoritative := false + triedFallback := false + did, dnsErr = d.ResolveHandleDNS(ctx, handle) + if errors.Is(dnsErr, ErrHandleNotFound) && d.TryAuthoritativeDNS { + slog.Debug("attempting authoritative handle DNS resolution", "handle", handle) + triedAuthoritative = true + // try harder with authoritative lookup + did, dnsErr = d.ResolveHandleDNSAuthoritative(ctx, handle) + } + if errors.Is(dnsErr, ErrHandleNotFound) && len(d.FallbackDNSServers) > 0 { + slog.Debug("attempting fallback DNS resolution", "handle", handle) + triedFallback = true + // try harder with fallback lookup + did, dnsErr = d.ResolveHandleDNSFallback(ctx, handle) + } + elapsed := time.Since(start) + slog.Debug("resolve handle DNS", "handle", handle, "err", dnsErr, "did", did, "authoritative", triedAuthoritative, "fallback", triedFallback, "duration_ms", elapsed.Milliseconds()) + if nil == dnsErr { // if *not* an error + return did, nil + } + } + + start := time.Now() + did, httpErr := d.ResolveHandleWellKnown(ctx, handle) + elapsed := time.Since(start) + slog.Debug("resolve handle HTTP well-known", "handle", handle, "err", httpErr, "did", did, "duration_ms", elapsed.Milliseconds()) + if nil == httpErr { // if *not* an error + return did, nil + } + + // return the most specific/helpful error + if !errors.Is(dnsErr, ErrHandleNotFound) { + return "", dnsErr + } + if !errors.Is(httpErr, ErrHandleNotFound) { + return "", httpErr + } + return "", dnsErr +} diff --git a/atproto/identity/identity.go b/atproto/identity/identity.go new file mode 100644 index 000000000..3dde38f8f --- /dev/null +++ b/atproto/identity/identity.go @@ -0,0 +1,209 @@ +package identity + +import ( + "fmt" + "net/url" + "strings" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/mr-tron/base58" +) + +// Represents an atproto identity. Could be a regular user account, or a service account (eg, feed generator) +type Identity struct { + DID syntax.DID + + // Handle/DID mapping must be bi-directionally verified. If that fails, the Handle should be the special 'handle.invalid' value + Handle syntax.Handle + + // These fields represent a parsed subset of a DID document. They are all nullable. Note that the services and keys maps do not preserve order, so they don't exactly round-trip DID documents. + AlsoKnownAs []string + Services map[string]ServiceEndpoint + Keys map[string]VerificationMethod +} + +// Sub-field type for [Identity], representing a cryptographic public key declared as a "verificationMethod" in the DID document. +type VerificationMethod struct { + Type string + PublicKeyMultibase string +} + +// Sub-field type for [Identity], representing a service endpoint URL declared in the DID document. +type ServiceEndpoint struct { + Type string + URL string +} + +// Extracts the information relevant to atproto from an arbitrary DID document. +// +// Always returns an invalid Handle field; calling code should only populate that field if it has been bi-directionally verified. +func ParseIdentity(doc *DIDDocument) Identity { + keys := make(map[string]VerificationMethod, len(doc.VerificationMethod)) + for _, vm := range doc.VerificationMethod { + parts := strings.SplitN(vm.ID, "#", 2) + if len(parts) < 2 { + continue + } + // ignore keys not controlled by this DID itself + if vm.Controller != doc.DID.String() { + continue + } + // don't want to clobber existing entries with same ID fragment + if _, ok := keys[parts[1]]; ok { + continue + } + // TODO: verify that ID and type match for atproto-specific services? + keys[parts[1]] = VerificationMethod{ + Type: vm.Type, + PublicKeyMultibase: vm.PublicKeyMultibase, + } + } + svc := make(map[string]ServiceEndpoint, len(doc.Service)) + for _, s := range doc.Service { + parts := strings.SplitN(s.ID, "#", 2) + if len(parts) < 2 { + continue + } + // don't want to clobber existing entries with same ID fragment + if _, ok := svc[parts[1]]; ok { + continue + } + // TODO: verify that ID and type match for atproto-specific services? + svc[parts[1]] = ServiceEndpoint{ + Type: s.Type, + URL: s.ServiceEndpoint, + } + } + return Identity{ + DID: doc.DID, + Handle: syntax.HandleInvalid, + AlsoKnownAs: doc.AlsoKnownAs, + Services: svc, + Keys: keys, + } +} + +// Helper to generate a DID document based on an identity. Note that there is flexibility around parsing, and this won't necessarily "round-trip" for every valid DID document. +func (ident *Identity) DIDDocument() DIDDocument { + doc := DIDDocument{ + DID: ident.DID, + AlsoKnownAs: ident.AlsoKnownAs, + VerificationMethod: make([]DocVerificationMethod, len(ident.Keys)), + Service: make([]DocService, len(ident.Services)), + } + i := 0 + for k, key := range ident.Keys { + doc.VerificationMethod[i] = DocVerificationMethod{ + ID: fmt.Sprintf("%s#%s", ident.DID, k), + Type: key.Type, + Controller: ident.DID.String(), + PublicKeyMultibase: key.PublicKeyMultibase, + } + i += 1 + } + i = 0 + for k, svc := range ident.Services { + doc.Service[i] = DocService{ + ID: fmt.Sprintf("#%s", k), + Type: svc.Type, + ServiceEndpoint: svc.URL, + } + i += 1 + } + return doc +} + +// Identifies and parses the atproto repo signing public key, specifically, out of any keys in this identity's DID document. +// +// Returns [ErrKeyNotFound] if there is no such key. +// +// Note that [atcrypto.PublicKey] is an interface, not a concrete type. +func (i *Identity) PublicKey() (atcrypto.PublicKey, error) { + return i.GetPublicKey("atproto") +} + +// Identifies and parses a specified service signing public key out of any keys in this identity's DID document. +// +// Returns [ErrKeyNotFound] if there is no such key. +// +// Note that [atcrypto.PublicKey] is an interface, not a concrete type. +func (i *Identity) GetPublicKey(id string) (atcrypto.PublicKey, error) { + if i.Keys == nil { + return nil, ErrKeyNotDeclared + } + k, ok := i.Keys[id] + if !ok { + return nil, ErrKeyNotDeclared + } + switch k.Type { + case "Multikey": + return atcrypto.ParsePublicMultibase(k.PublicKeyMultibase) + case "EcdsaSecp256r1VerificationKey2019": + if len(k.PublicKeyMultibase) < 2 || k.PublicKeyMultibase[0] != 'z' { + return nil, fmt.Errorf("identity key not a multibase base58btc string") + } + keyBytes, err := base58.Decode(k.PublicKeyMultibase[1:]) + if err != nil { + return nil, fmt.Errorf("identity key multibase parsing: %w", err) + } + return atcrypto.ParsePublicUncompressedBytesP256(keyBytes) + case "EcdsaSecp256k1VerificationKey2019": + if len(k.PublicKeyMultibase) < 2 || k.PublicKeyMultibase[0] != 'z' { + return nil, fmt.Errorf("identity key not a multibase base58btc string") + } + keyBytes, err := base58.Decode(k.PublicKeyMultibase[1:]) + if err != nil { + return nil, fmt.Errorf("identity key multibase parsing: %w", err) + } + return atcrypto.ParsePublicUncompressedBytesK256(keyBytes) + default: + return nil, fmt.Errorf("unsupported atproto public key type: %s", k.Type) + } +} + +// The home PDS endpoint for this identity, if one is included in the DID document. +// +// The endpoint should be an HTTP URL with method, hostname, and optional port. It may or may not include path segments. +// +// Returns an empty string if the service isn't found, or if the URL fails to parse. +func (i *Identity) PDSEndpoint() string { + return i.GetServiceEndpoint("atproto_pds") +} + +// Returns the service endpoint URL for specified service ID (the fragment part of identifier, not including the hash symbol). +// +// The endpoint should be an HTTP URL with method, hostname, and optional port. It may or may not include path segments. +// +// Returns an empty string if the service isn't found, or if the URL fails to parse. +func (i *Identity) GetServiceEndpoint(id string) string { + if i.Services == nil { + return "" + } + endpoint, ok := i.Services[id] + if !ok { + return "" + } + _, err := url.Parse(endpoint.URL) + if err != nil { + return "" + } + return endpoint.URL +} + +// Returns an atproto handle from the alsoKnownAs URI list for this identifier. Returns an error if there is no handle, or if an at:// URI fails to parse as a handle. +// +// Note that this handle is *not* necessarily to be trusted, as it may not have been bi-directionally verified. The 'Handle' field on the 'Identity' should contain either a verified handle, or the special 'handle.invalid' indicator value. +func (i *Identity) DeclaredHandle() (syntax.Handle, error) { + for _, u := range i.AlsoKnownAs { + if strings.HasPrefix(u, "at://") && len(u) > len("at://") { + hdl, err := syntax.ParseHandle(u[5:]) + if err != nil { + continue + } + return hdl.Normalize(), nil + } + } + return "", ErrHandleNotDeclared +} diff --git a/atproto/identity/identity_test.go b/atproto/identity/identity_test.go new file mode 100644 index 000000000..0512c0b70 --- /dev/null +++ b/atproto/identity/identity_test.go @@ -0,0 +1,70 @@ +package identity + +import ( + "encoding/json" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +// Tests parsing and normalizing handles from DID documents +func TestHandleExtraction(t *testing.T) { + assert := assert.New(t) + f, err := os.Open("testdata/did_plc_doc.json") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + docBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var doc DIDDocument + err = json.Unmarshal(docBytes, &doc) + assert.NoError(err) + + { + ident := ParseIdentity(&doc) + hdl, err := ident.DeclaredHandle() + assert.NoError(err) + assert.Equal("atproto.com", hdl.String()) + } + + { + doc.AlsoKnownAs = []string{ + "at://BLAH.com", + "at://other.org", + } + ident := ParseIdentity(&doc) + hdl, err := ident.DeclaredHandle() + assert.NoError(err) + assert.Equal("blah.com", hdl.String()) + } + + { + doc.AlsoKnownAs = []string{ + "https://http.example.com", + "at://under_example_com", + "at://correct.EXAMPLE.com", + "at://other.example.com", + } + ident := ParseIdentity(&doc) + hdl, err := ident.DeclaredHandle() + assert.NoError(err) + assert.Equal("correct.example.com", hdl.String()) + } + + { + doc.AlsoKnownAs = []string{ + "https://http.example.com", + } + ident := ParseIdentity(&doc) + _, err := ident.DeclaredHandle() + assert.Error(err) + assert.Equal("handle.invalid", ident.Handle.String()) + } +} diff --git a/atproto/identity/live_test.go b/atproto/identity/live_test.go new file mode 100644 index 000000000..c763bf098 --- /dev/null +++ b/atproto/identity/live_test.go @@ -0,0 +1,154 @@ +package identity + +import ( + "context" + "log/slog" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "golang.org/x/time/rate" + + "github.com/stretchr/testify/assert" +) + +// NOTE: this hits the open internet! marked as skip below by default +func testDirectoryLive(t *testing.T, d Directory) { + assert := assert.New(t) + ctx := context.Background() + + handle := syntax.Handle("atproto.com") + did := syntax.DID("did:plc:ewvi7nxzyoun6zhxrhs64oiz") + pdsSuffix := "host.bsky.network" + + resp, err := d.LookupHandle(ctx, handle) + assert.NoError(err) + assert.Equal(handle, resp.Handle) + assert.Equal(did, resp.DID) + assert.True(strings.HasSuffix(resp.PDSEndpoint(), pdsSuffix)) + dh, err := resp.DeclaredHandle() + assert.NoError(err) + assert.Equal(handle, dh) + pk, err := resp.PublicKey() + assert.NoError(err) + assert.NotNil(pk) + + resp, err = d.LookupDID(ctx, did) + assert.NoError(err) + assert.Equal(handle, resp.Handle) + assert.Equal(did, resp.DID) + assert.True(strings.HasSuffix(resp.PDSEndpoint(), pdsSuffix)) + + _, err = d.LookupHandle(ctx, syntax.Handle("fake-dummy-no-resolve.atproto.com")) + assert.ErrorIs(err, ErrHandleNotFound) + + _, err = d.LookupDID(ctx, syntax.DID("did:web:fake-dummy-no-resolve.atproto.com")) + assert.ErrorIs(err, ErrDIDNotFound) + + _, err = d.LookupDID(ctx, syntax.DID("did:plc:fake-dummy-no-resolve.atproto.com")) + assert.ErrorIs(err, ErrDIDNotFound) + + _, err = d.LookupHandle(ctx, syntax.HandleInvalid) + assert.Error(err) +} + +func TestBaseDirectory(t *testing.T) { + t.Skip("TODO: skipping live network test") + d := BaseDirectory{} + testDirectoryLive(t, &d) +} + +func TestCacheDirectory(t *testing.T) { + t.Skip("TODO: skipping live network test") + inner := BaseDirectory{} + d := NewCacheDirectory(&inner, 1000, time.Hour*1, time.Hour*1, time.Hour*1) + for i := 0; i < 3; i = i + 1 { + testDirectoryLive(t, &d) + } +} + +func TestCacheCoalesce(t *testing.T) { + t.Skip("TODO: skipping live network test") + + assert := assert.New(t) + handle := syntax.Handle("atproto.com") + did := syntax.DID("did:plc:ewvi7nxzyoun6zhxrhs64oiz") + + base := BaseDirectory{ + PLCURL: "https://plc.directory", + HTTPClient: http.Client{ + Timeout: time.Second * 15, + }, + // Limit the number of requests we can make to the PLC to 1 per second + PLCLimiter: rate.NewLimiter(1, 1), + TryAuthoritativeDNS: true, + SkipDNSDomainSuffixes: []string{".bsky.social"}, + } + dir := NewCacheDirectory(&base, 1000, time.Hour*1, time.Hour*1, time.Hour*1) + // All 60 routines launch at the same time, so they should all miss the cache initially + routines := 60 + wg := sync.WaitGroup{} + + // Cancel the context after 2 seconds, if we're coalescing correctly, we should only make 1 request + ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + defer cancel() + for i := 0; i < routines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ident, err := dir.LookupDID(ctx, did) + if err != nil { + slog.Error("Failed lookup", "error", err) + } + assert.NoError(err) + assert.Equal(handle, ident.Handle) + + ident, err = dir.LookupHandle(ctx, handle) + if err != nil { + slog.Error("Failed lookup", "error", err) + } + assert.NoError(err) + assert.Equal(did, ident.DID) + }() + } + wg.Wait() +} + +func TestFallbackDNS(t *testing.T) { + t.Skip("TODO: skipping live network test") + + assert := assert.New(t) + ctx := context.Background() + handle := syntax.Handle("no-such-record.atproto.com") + dir := BaseDirectory{ + FallbackDNSServers: []string{"1.1.1.1:53", "8.8.8.8:53"}, + } + + // valid DNS server + _, err := dir.LookupHandle(ctx, handle) + assert.Error(err) + assert.ErrorIs(err, ErrHandleNotFound) + + // invalid DNS server syntax + dir.FallbackDNSServers = []string{"_"} + _, err = dir.LookupHandle(ctx, handle) + assert.Error(err) + assert.ErrorIs(err, ErrHandleResolutionFailed) +} + +func TestResolveNSID(t *testing.T) { + t.Skip("TODO: skipping live network test") + assert := assert.New(t) + ctx := context.Background() + + dir := BaseDirectory{} + // NOTE: this was a very short temporary NSID when rkey restriction was short + nsid := syntax.NSID("net.bnewbold.m") + did, err := dir.ResolveNSID(ctx, nsid) + + assert.NoError(err) + assert.Equal(did, syntax.DID("did:plc:nhxcyu4ewwhl5pqil4dotqjo")) +} diff --git a/atproto/identity/metrics.go b/atproto/identity/metrics.go new file mode 100644 index 000000000..203e7039a --- /dev/null +++ b/atproto/identity/metrics.go @@ -0,0 +1,28 @@ +package identity + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var handleResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_resolve_handle", + Help: "ATProto handle resolutions", +}, []string{"directory", "status"}) + +var handleResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_resolve_handle_duration", + Help: "Time to resolve a handle", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) + +var didResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_resolve_did", + Help: "ATProto DID resolutions", +}, []string{"directory", "status"}) + +var didResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_resolve_did_duration", + Help: "Time to resolve a DID", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) diff --git a/atproto/identity/metrics_legacy.go b/atproto/identity/metrics_legacy.go new file mode 100644 index 000000000..3f2c2eae1 --- /dev/null +++ b/atproto/identity/metrics_legacy.go @@ -0,0 +1,42 @@ +package identity + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// DEPRECATED +var handleCacheHits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_handle_cache_hits", + Help: "Number of cache hits for ATProto handle lookups", +}) + +// DEPRECATED +var handleCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_handle_cache_misses", + Help: "Number of cache misses for ATProto handle lookups", +}) + +// DEPRECATED +var identityCacheHits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_identity_cache_hits", + Help: "Number of cache hits for ATProto identity lookups", +}) + +// DEPRECATED +var identityCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_identity_cache_misses", + Help: "Number of cache misses for ATProto identity lookups", +}) + +// DEPRECATED +var identityRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_identity_requests_coalesced", + Help: "Number of identity requests coalesced", +}) + +// DEPRECATED +var handleRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_directory_handle_requests_coalesced", + Help: "Number of handle requests coalesced", +}) diff --git a/atproto/identity/mock_directory.go b/atproto/identity/mock_directory.go new file mode 100644 index 000000000..845395578 --- /dev/null +++ b/atproto/identity/mock_directory.go @@ -0,0 +1,123 @@ +package identity + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// A fake identity directory, for use in tests +type MockDirectory struct { + mu *sync.RWMutex + Handles map[syntax.Handle]syntax.DID + Identities map[syntax.DID]Identity +} + +var _ Directory = (*MockDirectory)(nil) +var _ Resolver = (*MockDirectory)(nil) + +func NewMockDirectory() MockDirectory { + return MockDirectory{ + mu: &sync.RWMutex{}, + Handles: make(map[syntax.Handle]syntax.DID), + Identities: make(map[syntax.DID]Identity), + } +} + +func (d *MockDirectory) Insert(ident Identity) { + d.mu.Lock() + defer d.mu.Unlock() + + if !ident.Handle.IsInvalidHandle() { + d.Handles[ident.Handle.Normalize()] = ident.DID + } + d.Identities[ident.DID] = ident +} + +func (d *MockDirectory) LookupHandle(ctx context.Context, h syntax.Handle) (*Identity, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + h = h.Normalize() + did, ok := d.Handles[h] + if !ok { + return nil, ErrHandleNotFound + } + ident, ok := d.Identities[did] + if !ok { + return nil, ErrDIDNotFound + } + return &ident, nil +} + +func (d *MockDirectory) LookupDID(ctx context.Context, did syntax.DID) (*Identity, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + ident, ok := d.Identities[did] + if !ok { + return nil, ErrDIDNotFound + } + return &ident, nil +} + +func (d *MockDirectory) Lookup(ctx context.Context, a syntax.AtIdentifier) (*Identity, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + handle, err := a.AsHandle() + if nil == err { // if not an error, is a Handle + return d.LookupHandle(ctx, handle) + } + did, err := a.AsDID() + if nil == err { // if not an error, is a DID + return d.LookupDID(ctx, did) + } + return nil, fmt.Errorf("at-identifier neither a Handle nor a DID") +} + +func (d *MockDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + h = h.Normalize() + did, ok := d.Handles[h] + if !ok { + return "", ErrHandleNotFound + } + return did, nil +} + +func (d *MockDirectory) ResolveDID(ctx context.Context, did syntax.DID) (*DIDDocument, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + ident, ok := d.Identities[did] + if !ok { + return nil, ErrDIDNotFound + } + doc := ident.DIDDocument() + return &doc, nil +} + +func (d *MockDirectory) ResolveDIDRaw(ctx context.Context, did syntax.DID) (json.RawMessage, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + ident, ok := d.Identities[did] + if !ok { + return nil, ErrDIDNotFound + } + doc := ident.DIDDocument() + return json.Marshal(doc) +} + +func (d *MockDirectory) Purge(ctx context.Context, a syntax.AtIdentifier) error { + d.mu.Lock() + defer d.mu.Unlock() + + return nil +} diff --git a/atproto/identity/mock_directory_test.go b/atproto/identity/mock_directory_test.go new file mode 100644 index 000000000..1802a6a65 --- /dev/null +++ b/atproto/identity/mock_directory_test.go @@ -0,0 +1,73 @@ +package identity + +import ( + "context" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" +) + +func TestMockDirectory(t *testing.T) { + var err error + assert := assert.New(t) + ctx := context.Background() + c := NewMockDirectory() + id1 := Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + } + id2 := Identity{ + DID: syntax.DID("did:plc:abc222"), + Handle: syntax.HandleInvalid, + } + id3 := Identity{ + DID: syntax.DID("did:plc:abc333"), + Handle: syntax.Handle("handle3.example.com"), + } + + // first, empty directory + _, err = c.LookupHandle(ctx, syntax.Handle("handle.example.com")) + assert.ErrorIs(err, ErrHandleNotFound) + _, err = c.LookupDID(ctx, syntax.DID("did:plc:abc123")) + assert.ErrorIs(err, ErrDIDNotFound) + + c.Insert(id1) + c.Insert(id2) + c.Insert(id3) + + out, err := c.LookupHandle(ctx, syntax.Handle("handle.example.com")) + assert.NoError(err) + assert.Equal(&id1, out) + out, err = c.LookupDID(ctx, syntax.DID("did:plc:abc111")) + assert.NoError(err) + assert.Equal(&id1, out) + + out, err = c.LookupDID(ctx, syntax.DID("did:plc:abc222")) + assert.NoError(err) + assert.True(out.Handle.IsInvalidHandle()) + + _, err = c.LookupHandle(ctx, syntax.HandleInvalid) + assert.ErrorIs(err, ErrHandleNotFound) + _, err = c.LookupDID(ctx, syntax.DID("did:plc:abc999")) + assert.ErrorIs(err, ErrDIDNotFound) + + did, err := c.ResolveHandle(ctx, syntax.Handle("handle.example.com")) + assert.NoError(err) + assert.Equal(id1.DID, did) + _, err = c.ResolveHandle(ctx, syntax.Handle("notfound.example.com")) + assert.ErrorIs(err, ErrHandleNotFound) + + _, err = c.ResolveDID(ctx, syntax.DID("did:plc:abc222")) + assert.NoError(err) + // TODO: verify structure matches + + _, err = c.ResolveDID(ctx, syntax.DID("did:plc:abc999")) + assert.ErrorIs(err, ErrDIDNotFound) + + // handle lookups should be case-insensitive + _, err = c.ResolveHandle(ctx, syntax.Handle("handle.EXAMPLE.com")) + assert.NoError(err) + assert.Equal(id1.DID, did) +} diff --git a/atproto/identity/nsid.go b/atproto/identity/nsid.go new file mode 100644 index 000000000..567d0cf66 --- /dev/null +++ b/atproto/identity/nsid.go @@ -0,0 +1,36 @@ +package identity + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +var ( + ErrNSIDResolutionFailed = fmt.Errorf("NSID resolution mechanism failed") + ErrNSIDNotFound = fmt.Errorf("NSID not associated with a DID") +) + +// Resolves an NSID to a DID, as used for Lexicon resolution (using "_lexicon" DNS TXT record) +func (d *BaseDirectory) ResolveNSID(ctx context.Context, nsid syntax.NSID) (syntax.DID, error) { + + domain := nsid.Authority() + res, err := d.Resolver.LookupTXT(ctx, "_lexicon."+domain) + + // check for NXDOMAIN + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsNotFound { + return "", ErrNSIDNotFound + } + } + + if err != nil { + return "", fmt.Errorf("%w: %w", ErrNSIDResolutionFailed, err) + } + + return parseTXTResp(res) +} diff --git a/atproto/identity/redisdir/doc.go b/atproto/identity/redisdir/doc.go new file mode 100644 index 000000000..5cab1d5eb --- /dev/null +++ b/atproto/identity/redisdir/doc.go @@ -0,0 +1,4 @@ +/* +Identity Directory implementation with tiered caching, using Redis. +*/ +package redisdir diff --git a/atproto/identity/redisdir/live_test.go b/atproto/identity/redisdir/live_test.go new file mode 100644 index 000000000..433c45b8c --- /dev/null +++ b/atproto/identity/redisdir/live_test.go @@ -0,0 +1,136 @@ +package redisdir + +import ( + "context" + "log/slog" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "golang.org/x/time/rate" + + "github.com/stretchr/testify/assert" +) + +var redisLocalTestURL string = "redis://localhost:6379/0" + +// NOTE: this hits the open internet! marked as skip below by default +func testDirectoryLive(t *testing.T, d identity.Directory) { + assert := assert.New(t) + ctx := context.Background() + + handle := syntax.Handle("atproto.com") + did := syntax.DID("did:plc:ewvi7nxzyoun6zhxrhs64oiz") + pdsSuffix := "host.bsky.network" + + resp, err := d.LookupHandle(ctx, handle) + assert.NoError(err) + assert.Equal(handle, resp.Handle) + assert.Equal(did, resp.DID) + assert.True(strings.HasSuffix(resp.PDSEndpoint(), pdsSuffix)) + dh, err := resp.DeclaredHandle() + assert.NoError(err) + assert.Equal(handle, dh) + pk, err := resp.PublicKey() + assert.NoError(err) + assert.NotNil(pk) + + resp, err = d.LookupDID(ctx, did) + assert.NoError(err) + assert.Equal(handle, resp.Handle) + assert.Equal(did, resp.DID) + assert.True(strings.HasSuffix(resp.PDSEndpoint(), pdsSuffix)) + + _, err = d.LookupHandle(ctx, syntax.Handle("fake-dummy-no-resolve.atproto.com")) + assert.Error(err) + //assert.ErrorIs(err, identity.ErrHandleNotFound) + + _, err = d.LookupDID(ctx, syntax.DID("did:web:fake-dummy-no-resolve.atproto.com")) + assert.Error(err) + //assert.ErrorIs(err, identity.ErrDIDNotFound) + + _, err = d.LookupDID(ctx, syntax.DID("did:plc:fake-dummy-no-resolve.atproto.com")) + assert.Error(err) + //assert.ErrorIs(err, identity.ErrDIDNotFound) + + _, err = d.LookupHandle(ctx, syntax.HandleInvalid) + assert.Error(err) +} + +func TestRedisDirectory(t *testing.T) { + t.Skip("TODO: skipping live network test") + assert := assert.New(t) + ctx := context.Background() + inner := identity.BaseDirectory{} + d, err := NewRedisDirectory(&inner, redisLocalTestURL, time.Hour*1, time.Hour*1, time.Hour*1, 1000) + if err != nil { + t.Fatal(err) + } + + err = d.Purge(ctx, syntax.Handle("atproto.com").AtIdentifier()) + assert.NoError(err) + err = d.Purge(ctx, syntax.Handle("fake-dummy-no-resolve.atproto.com").AtIdentifier()) + assert.NoError(err) + err = d.Purge(ctx, syntax.DID("did:web:fake-dummy-no-resolve.atproto.com").AtIdentifier()) + assert.NoError(err) + err = d.Purge(ctx, syntax.DID("did:plc:fake-dummy-no-resolve.atproto.com").AtIdentifier()) + assert.NoError(err) + + for i := 0; i < 3; i = i + 1 { + testDirectoryLive(t, d) + } +} + +func TestRedisCoalesce(t *testing.T) { + t.Skip("TODO: skipping live network test") + + assert := assert.New(t) + handle := syntax.Handle("atproto.com") + did := syntax.DID("did:plc:ewvi7nxzyoun6zhxrhs64oiz") + + base := identity.BaseDirectory{ + PLCURL: "https://plc.directory", + HTTPClient: http.Client{ + Timeout: time.Second * 15, + }, + // Limit the number of requests we can make to the PLC to 1 per second + PLCLimiter: rate.NewLimiter(1, 1), + TryAuthoritativeDNS: true, + SkipDNSDomainSuffixes: []string{".bsky.social"}, + } + dir, err := NewRedisDirectory(&base, redisLocalTestURL, time.Hour*1, time.Hour*1, time.Hour*1, 1000) + if err != nil { + t.Fatal(err) + } + // All 60 routines launch at the same time, so they should all miss the cache initially + routines := 60 + wg := sync.WaitGroup{} + + // Cancel the context after 2 seconds, if we're coalescing correctly, we should only make 1 request + ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) + defer cancel() + for i := 0; i < routines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ident, err := dir.LookupDID(ctx, did) + if err != nil { + slog.Error("Failed lookup", "error", err) + } + assert.NoError(err) + assert.Equal(handle, ident.Handle) + + ident, err = dir.LookupHandle(ctx, handle) + if err != nil { + slog.Error("Failed lookup", "error", err) + } + assert.NoError(err) + assert.Equal(did, ident.DID) + }() + } + wg.Wait() +} diff --git a/atproto/identity/redisdir/metrics.go b/atproto/identity/redisdir/metrics.go new file mode 100644 index 000000000..f19036174 --- /dev/null +++ b/atproto/identity/redisdir/metrics.go @@ -0,0 +1,28 @@ +package redisdir + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var handleResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_redisdir_resolve_handle", + Help: "ATProto handle resolutions", +}, []string{"directory", "status"}) + +var handleResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_redisdir_resolve_handle_duration", + Help: "Time to resolve a handle", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) + +var didResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_redisdir_resolve_did", + Help: "ATProto DID resolutions", +}, []string{"directory", "status"}) + +var didResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_redisdir_resolve_did_duration", + Help: "Time to resolve a DID", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) diff --git a/atproto/identity/redisdir/metrics_legacy.go b/atproto/identity/redisdir/metrics_legacy.go new file mode 100644 index 000000000..f107c1054 --- /dev/null +++ b/atproto/identity/redisdir/metrics_legacy.go @@ -0,0 +1,42 @@ +package redisdir + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// DEPRECATED +var handleCacheHits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_handle_cache_hits", + Help: "Number of cache hits for ATProto handle lookups", +}) + +// DEPRECATED +var handleCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_handle_cache_misses", + Help: "Number of cache misses for ATProto handle lookups", +}) + +// DEPRECATED +var identityCacheHits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_identity_cache_hits", + Help: "Number of cache hits for ATProto identity lookups", +}) + +// DEPRECATED +var identityCacheMisses = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_identity_cache_misses", + Help: "Number of cache misses for ATProto identity lookups", +}) + +// DEPRECATED +var identityRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_identity_requests_coalesced", + Help: "Number of identity requests coalesced", +}) + +// DEPRECATED +var handleRequestsCoalesced = promauto.NewCounter(prometheus.CounterOpts{ + Name: "atproto_redis_directory_handle_requests_coalesced", + Help: "Number of handle requests coalesced", +}) diff --git a/atproto/identity/redisdir/redis_directory.go b/atproto/identity/redisdir/redis_directory.go new file mode 100644 index 000000000..af9f4863f --- /dev/null +++ b/atproto/identity/redisdir/redis_directory.go @@ -0,0 +1,401 @@ +package redisdir + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/go-redis/cache/v9" + "github.com/redis/go-redis/v9" +) + +// prefix string for all the Redis keys this cache uses +var redisDirPrefix string = "dir/" + +// Uses redis as a cache for identity lookups. +// +// Includes an in-process LRU cache as well (provided by the redis client library), for hot key (identities). +type RedisDirectory struct { + Inner identity.Directory + ErrTTL time.Duration + HitTTL time.Duration + InvalidHandleTTL time.Duration + + handleCache *cache.Cache + identityCache *cache.Cache + didLookupChans sync.Map + handleLookupChans sync.Map +} + +type handleEntry struct { + Updated time.Time + // needs to be pointer type, because unmarshalling empty string would be an error + DID *syntax.DID + Err error +} + +type identityEntry struct { + Updated time.Time + Identity *identity.Identity + Err error +} + +var _ identity.Directory = (*RedisDirectory)(nil) + +// Creates a new caching `identity.Directory` wrapper around an existing directory, using Redis and in-process LRU for caching. +// +// `redisURL` contains all the redis connection config options. +// `hitTTL` and `errTTL` define how long successful and errored identity metadata should be cached (respectively). errTTL is expected to be shorted than hitTTL. +// `lruSize` is the size of the in-process cache, for each of the handle and identity caches. 10000 is a reasonable default. +// +// NOTE: Errors returned may be inconsistent with the base directory, or between calls. This is because cached errors are serialized/deserialized and that may break equality checks. +func NewRedisDirectory(inner identity.Directory, redisURL string, hitTTL, errTTL, invalidHandleTTL time.Duration, lruSize int) (*RedisDirectory, error) { + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, fmt.Errorf("could not configure redis identity cache: %w", err) + } + rdb := redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(context.TODO()).Result() + if err != nil { + return nil, fmt.Errorf("could not connect to redis identity cache: %w", err) + } + handleCache := cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(lruSize, hitTTL), + }) + identityCache := cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(lruSize, hitTTL), + }) + return &RedisDirectory{ + Inner: inner, + ErrTTL: errTTL, + HitTTL: hitTTL, + InvalidHandleTTL: invalidHandleTTL, + handleCache: handleCache, + identityCache: identityCache, + }, nil +} + +func (d *RedisDirectory) isHandleStale(e *handleEntry) bool { + if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { + return true + } + return false +} + +func (d *RedisDirectory) isIdentityStale(e *identityEntry) bool { + if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { + return true + } + if e.Identity != nil && e.Identity.Handle.IsInvalidHandle() && time.Since(e.Updated) > d.InvalidHandleTTL { + return true + } + return false +} + +func (d *RedisDirectory) updateHandle(ctx context.Context, h syntax.Handle) handleEntry { + h = h.Normalize() + ident, err := d.Inner.LookupHandle(ctx, h) + if err != nil { + he := handleEntry{ + Updated: time.Now(), + DID: nil, + Err: err, + } + err = d.handleCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + h.String(), + Value: he, + TTL: d.ErrTTL, + }) + if err != nil { + slog.Error("identity cache write failed", "cache", "handle", "err", err) + } + return he + } + + entry := identityEntry{ + Updated: time.Now(), + Identity: ident, + Err: nil, + } + he := handleEntry{ + Updated: time.Now(), + DID: &ident.DID, + Err: nil, + } + + err = d.identityCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + ident.DID.String(), + Value: entry, + TTL: d.HitTTL, + }) + if err != nil { + slog.Error("identity cache write failed", "cache", "did", "did", ident.DID, "err", err) + } + err = d.handleCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + h.String(), + Value: he, + TTL: d.HitTTL, + }) + if err != nil { + slog.Error("identity cache write failed", "cache", "handle", "did", ident.DID, "err", err) + } + return he +} + +func (d *RedisDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { + start := time.Now() + h = h.Normalize() + if h.IsInvalidHandle() { + return "", fmt.Errorf("can not resolve handle: %w", identity.ErrInvalidHandle) + } + var entry handleEntry + err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + handleResolution.WithLabelValues("redisdir", "error").Inc() + handleResolutionDuration.WithLabelValues("redisdir", "error").Observe(time.Since(start).Seconds()) + return "", fmt.Errorf("identity cache read failed: %w", err) + } + if err == nil && !d.isHandleStale(&entry) { // if no error... + handleCacheHits.Inc() + handleResolution.WithLabelValues("redisdir", "cached").Inc() + handleResolutionDuration.WithLabelValues("redisdir", "cached").Observe(time.Since(start).Seconds()) + if entry.Err != nil { + return "", entry.Err + } else if entry.DID != nil { + return *entry.DID, nil + } else { + return "", errors.New("code flow error in redis identity directory") + } + } + handleCacheMisses.Inc() + + // Coalesce multiple requests for the same Handle + res := make(chan struct{}) + val, loaded := d.handleLookupChans.LoadOrStore(h.String(), res) + if loaded { + handleRequestsCoalesced.Inc() + handleResolution.WithLabelValues("redisdir", "coalesced").Inc() + handleResolutionDuration.WithLabelValues("redisdir", "coalesced").Observe(time.Since(start).Seconds()) + // Wait for the result from the pending request + select { + case <-val.(chan struct{}): + // The result should now be in the cache + err := d.handleCache.Get(ctx, redisDirPrefix+h.String(), entry) + if err != nil && err != cache.ErrCacheMiss { + return "", fmt.Errorf("identity cache read failed: %w", err) + } + if err == nil && !d.isHandleStale(&entry) { // if no error... + if entry.Err != nil { + return "", entry.Err + } else if entry.DID != nil { + return *entry.DID, nil + } else { + return "", errors.New("code flow error in redis identity directory") + } + } + return "", errors.New("identity not found in cache after coalesce returned") + case <-ctx.Done(): + return "", ctx.Err() + } + } + + // Update the Handle Entry from PLC and cache the result + newEntry := d.updateHandle(ctx, h) + + // Cleanup the coalesce map and close the results channel + d.handleLookupChans.Delete(h.String()) + // Callers waiting will now get the result from the cache + close(res) + + if newEntry.Err != nil { + handleResolution.WithLabelValues("redisdir", "error").Inc() + handleResolutionDuration.WithLabelValues("redisdir", "error").Observe(time.Since(start).Seconds()) + return "", newEntry.Err + } + if newEntry.DID != nil { + handleResolution.WithLabelValues("redisdir", "success").Inc() + handleResolutionDuration.WithLabelValues("redisdir", "success").Observe(time.Since(start).Seconds()) + return *newEntry.DID, nil + } + return "", errors.New("unexpected control-flow error") +} + +func (d *RedisDirectory) updateDID(ctx context.Context, did syntax.DID) identityEntry { + ident, err := d.Inner.LookupDID(ctx, did) + // persist the identity lookup error, instead of processing it immediately + entry := identityEntry{ + Updated: time.Now(), + Identity: ident, + Err: err, + } + var he *handleEntry + // if *not* an error, then also update the handle cache + if err == nil && !ident.Handle.IsInvalidHandle() { + he = &handleEntry{ + Updated: time.Now(), + DID: &did, + Err: nil, + } + } + + err = d.identityCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + did.String(), + Value: entry, + TTL: d.HitTTL, + }) + if err != nil { + slog.Error("identity cache write failed", "cache", "did", "did", did, "err", err) + } + if he != nil { + err = d.handleCache.Set(&cache.Item{ + Ctx: ctx, + Key: redisDirPrefix + ident.Handle.String(), + Value: *he, + TTL: d.HitTTL, + }) + if err != nil { + slog.Error("identity cache write failed", "cache", "handle", "did", did, "err", err) + } + } + return entry +} + +func (d *RedisDirectory) LookupDID(ctx context.Context, did syntax.DID) (*identity.Identity, error) { + id, _, err := d.LookupDIDWithCacheState(ctx, did) + return id, err +} + +func (d *RedisDirectory) LookupDIDWithCacheState(ctx context.Context, did syntax.DID) (*identity.Identity, bool, error) { + start := time.Now() + var entry identityEntry + err := d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + didResolution.WithLabelValues("redisdir", "error").Inc() + didResolutionDuration.WithLabelValues("redisdir", "error").Observe(time.Since(start).Seconds()) + return nil, false, fmt.Errorf("identity cache read failed: %w", err) + } + if err == nil && !d.isIdentityStale(&entry) { // if no error... + identityCacheHits.Inc() + didResolution.WithLabelValues("redisdir", "cached").Inc() + didResolutionDuration.WithLabelValues("redisdir", "cached").Observe(time.Since(start).Seconds()) + return entry.Identity, true, entry.Err + } + identityCacheMisses.Inc() + + // Coalesce multiple requests for the same DID + res := make(chan struct{}) + val, loaded := d.didLookupChans.LoadOrStore(did.String(), res) + if loaded { + identityRequestsCoalesced.Inc() + didResolution.WithLabelValues("redisdir", "coalesced").Inc() + didResolutionDuration.WithLabelValues("redisdir", "coalesced").Observe(time.Since(start).Seconds()) + // Wait for the result from the pending request + select { + case <-val.(chan struct{}): + // The result should now be in the cache + err = d.identityCache.Get(ctx, redisDirPrefix+did.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + return nil, false, fmt.Errorf("identity cache read failed: %w", err) + } + if err == nil && !d.isIdentityStale(&entry) { // if no error... + return entry.Identity, false, entry.Err + } + return nil, false, errors.New("identity not found in cache after coalesce returned") + case <-ctx.Done(): + return nil, false, ctx.Err() + } + } + + // Update the Identity Entry from PLC and cache the result + newEntry := d.updateDID(ctx, did) + + // Cleanup the coalesce map and close the results channel + d.didLookupChans.Delete(did.String()) + // Callers waiting will now get the result from the cache + close(res) + + if newEntry.Err != nil { + didResolution.WithLabelValues("redisdir", "error").Inc() + didResolutionDuration.WithLabelValues("redisdir", "error").Observe(time.Since(start).Seconds()) + return nil, false, newEntry.Err + } + if newEntry.Identity != nil { + didResolution.WithLabelValues("redisdir", "success").Inc() + didResolutionDuration.WithLabelValues("redisdir", "success").Observe(time.Since(start).Seconds()) + return newEntry.Identity, false, nil + } + return nil, false, errors.New("unexpected control-flow error") +} + +func (d *RedisDirectory) LookupHandle(ctx context.Context, h syntax.Handle) (*identity.Identity, error) { + ident, _, err := d.LookupHandleWithCacheState(ctx, h) + return ident, err +} + +func (d *RedisDirectory) LookupHandleWithCacheState(ctx context.Context, h syntax.Handle) (*identity.Identity, bool, error) { + h = h.Normalize() + did, err := d.ResolveHandle(ctx, h) + if err != nil { + return nil, false, err + } + ident, hit, err := d.LookupDIDWithCacheState(ctx, did) + if err != nil { + return nil, hit, err + } + + declared, err := ident.DeclaredHandle() + if err != nil { + return nil, hit, err + } + // NOTE: DeclaredHandle() returns a normalized handle, and we already normalized 'h' above + if declared != h { + return nil, hit, identity.ErrHandleMismatch + } + return ident, hit, nil +} + +func (d *RedisDirectory) Lookup(ctx context.Context, a syntax.AtIdentifier) (*identity.Identity, error) { + handle, err := a.AsHandle() + if err == nil { // if not an error, is a handle + return d.LookupHandle(ctx, handle) + } + did, err := a.AsDID() + if err == nil { // if not an error, is a DID + return d.LookupDID(ctx, did) + } + return nil, errors.New("at-identifier neither a Handle nor a DID") +} + +func (d *RedisDirectory) Purge(ctx context.Context, a syntax.AtIdentifier) error { + handle, err := a.AsHandle() + if err == nil { // if not an error, is a handle + handle = handle.Normalize() + err = d.handleCache.Delete(ctx, redisDirPrefix+handle.String()) + if err == cache.ErrCacheMiss { + return nil + } + return err + } + did, err := a.AsDID() + if err == nil { // if not an error, is a DID + err = d.identityCache.Delete(ctx, redisDirPrefix+did.String()) + if err == cache.ErrCacheMiss { + return nil + } + return err + } + return errors.New("at-identifier neither a Handle nor a DID") +} diff --git a/atproto/identity/resolver.go b/atproto/identity/resolver.go new file mode 100644 index 000000000..47c162b1a --- /dev/null +++ b/atproto/identity/resolver.go @@ -0,0 +1,17 @@ +package identity + +import ( + "context" + "encoding/json" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Low-level interface for resolving DIDs and atproto handles. +// +// Most atproto code should use the `identity.Directory` interface instead. +type Resolver interface { + ResolveDID(ctx context.Context, did syntax.DID) (*DIDDocument, error) + ResolveDIDRaw(ctx context.Context, did syntax.DID) (json.RawMessage, error) + ResolveHandle(ctx context.Context, handle syntax.Handle) (syntax.DID, error) +} diff --git a/atproto/identity/testdata/did_plc_doc.json b/atproto/identity/testdata/did_plc_doc.json new file mode 100644 index 000000000..408a5e9bf --- /dev/null +++ b/atproto/identity/testdata/did_plc_doc.json @@ -0,0 +1,26 @@ +{ + "@context": [ + "https://www.w3.org/ns/did/v1", + "https://w3id.org/security/multikey/v1", + "https://w3id.org/security/suites/secp256k1-2019/v1" + ], + "id": "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "alsoKnownAs": [ + "at://atproto.com" + ], + "verificationMethod": [ + { + "id": "did:plc:ewvi7nxzyoun6zhxrhs64oiz#atproto", + "type": "Multikey", + "controller": "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "publicKeyMultibase": "zQ3shXjHeiBuRCKmM36cuYnm7YEMzhGnCmCyW92sRJ9pribSF" + } + ], + "service": [ + { + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://bsky.social" + } + ] +} diff --git a/atproto/identity/testdata/did_plc_doc_legacy.json b/atproto/identity/testdata/did_plc_doc_legacy.json new file mode 100644 index 000000000..df5f1656c --- /dev/null +++ b/atproto/identity/testdata/did_plc_doc_legacy.json @@ -0,0 +1,25 @@ +{ + "@context": [ + "https://www.w3.org/ns/did/v1", + "https://w3id.org/security/suites/secp256k1-2019/v1" + ], + "id": "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "alsoKnownAs": [ + "at://atproto.com" + ], + "verificationMethod": [ + { + "id": "#atproto", + "type": "EcdsaSecp256k1VerificationKey2019", + "controller": "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "publicKeyMultibase": "zQYEBzXeuTM9UR3rfvNag6L3RNAs5pQZyYPsomTsgQhsxLdEgCrPTLgFna8yqCnxPpNT7DBk6Ym3dgPKNu86vt9GR" + } + ], + "service": [ + { + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://bsky.social" + } + ] +} diff --git a/atproto/identity/testdata/did_web_doc.json b/atproto/identity/testdata/did_web_doc.json new file mode 100644 index 000000000..aef9dad5d --- /dev/null +++ b/atproto/identity/testdata/did_web_doc.json @@ -0,0 +1,16 @@ +{ + "@context": [ + "https://www.w3.org/ns/did/v1" + ], + "id": "did:web:discover.bsky.social", + "alsoKnownAs": [ ], + "authentication": null, + "verificationMethod": [ ], + "service": [ + { + "id": "#bsky_fg", + "type": "BskyFeedGenerator", + "serviceEndpoint": "https://discover.bsky.social" + } + ] +} diff --git a/atproto/labeling/cbor_gen.go b/atproto/labeling/cbor_gen.go new file mode 100644 index 000000000..8c3b2074e --- /dev/null +++ b/atproto/labeling/cbor_gen.go @@ -0,0 +1,503 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package labeling + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *Label) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 9 + + if t.CID == nil { + fieldCount-- + } + + if t.ExpiresAt == nil { + fieldCount-- + } + + if t.Negated == nil { + fieldCount-- + } + + if t.Sig == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.CID (string) (string) + if t.CID != nil { + + if len("cid") > 1000000 { + return xerrors.Errorf("Value in field \"cid\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil { + return err + } + if _, err := cw.WriteString(string("cid")); err != nil { + return err + } + + if t.CID == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.CID) > 1000000 { + return xerrors.Errorf("Value in field t.CID was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.CID))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.CID)); err != nil { + return err + } + } + } + + // t.CreatedAt (string) (string) + if len("cts") > 1000000 { + return xerrors.Errorf("Value in field \"cts\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cts"))); err != nil { + return err + } + if _, err := cw.WriteString(string("cts")); err != nil { + return err + } + + if len(t.CreatedAt) > 1000000 { + return xerrors.Errorf("Value in field t.CreatedAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.CreatedAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { + return err + } + + // t.ExpiresAt (string) (string) + if t.ExpiresAt != nil { + + if len("exp") > 1000000 { + return xerrors.Errorf("Value in field \"exp\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("exp"))); err != nil { + return err + } + if _, err := cw.WriteString(string("exp")); err != nil { + return err + } + + if t.ExpiresAt == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.ExpiresAt) > 1000000 { + return xerrors.Errorf("Value in field t.ExpiresAt was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.ExpiresAt))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.ExpiresAt)); err != nil { + return err + } + } + } + + // t.Negated (bool) (bool) + if t.Negated != nil { + + if len("neg") > 1000000 { + return xerrors.Errorf("Value in field \"neg\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("neg"))); err != nil { + return err + } + if _, err := cw.WriteString(string("neg")); err != nil { + return err + } + + if t.Negated == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteBool(w, *t.Negated); err != nil { + return err + } + } + } + + // t.Sig (atdata.Bytes) (slice) + if t.Sig != nil { + + if len("sig") > 1000000 { + return xerrors.Errorf("Value in field \"sig\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sig"))); err != nil { + return err + } + if _, err := cw.WriteString(string("sig")); err != nil { + return err + } + + if len(t.Sig) > 2097152 { + return xerrors.Errorf("Byte array in field t.Sig was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Sig))); err != nil { + return err + } + + if _, err := cw.Write(t.Sig); err != nil { + return err + } + + } + + // t.SourceDID (string) (string) + if len("src") > 1000000 { + return xerrors.Errorf("Value in field \"src\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("src"))); err != nil { + return err + } + if _, err := cw.WriteString(string("src")); err != nil { + return err + } + + if len(t.SourceDID) > 1000000 { + return xerrors.Errorf("Value in field t.SourceDID was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.SourceDID))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.SourceDID)); err != nil { + return err + } + + // t.URI (string) (string) + if len("uri") > 1000000 { + return xerrors.Errorf("Value in field \"uri\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { + return err + } + if _, err := cw.WriteString(string("uri")); err != nil { + return err + } + + if len(t.URI) > 1000000 { + return xerrors.Errorf("Value in field t.URI was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.URI))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.URI)); err != nil { + return err + } + + // t.Val (string) (string) + if len("val") > 1000000 { + return xerrors.Errorf("Value in field \"val\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("val"))); err != nil { + return err + } + if _, err := cw.WriteString(string("val")); err != nil { + return err + } + + if len(t.Val) > 1000000 { + return xerrors.Errorf("Value in field t.Val was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Val))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Val)); err != nil { + return err + } + + // t.Version (int64) (int64) + if len("ver") > 1000000 { + return xerrors.Errorf("Value in field \"ver\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ver"))); err != nil { + return err + } + if _, err := cw.WriteString(string("ver")); err != nil { + return err + } + + if t.Version >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Version)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Version-1)); err != nil { + return err + } + } + + return nil +} + +func (t *Label) UnmarshalCBOR(r io.Reader) (err error) { + *t = Label{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Label: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 3) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.CID (string) (string) + case "cid": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CID = (*string)(&sval) + } + } + // t.CreatedAt (string) (string) + case "cts": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.CreatedAt = string(sval) + } + // t.ExpiresAt (string) (string) + case "exp": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.ExpiresAt = (*string)(&sval) + } + } + // t.Negated (bool) (bool) + case "neg": + + { + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + + var val bool + switch extra { + case 20: + val = false + case 21: + val = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + t.Negated = &val + } + } + // t.Sig (atdata.Bytes) (slice) + case "sig": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.Sig: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Sig = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Sig); err != nil { + return err + } + + // t.SourceDID (string) (string) + case "src": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.SourceDID = string(sval) + } + // t.URI (string) (string) + case "uri": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.URI = string(sval) + } + // t.Val (string) (string) + case "val": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Val = string(sval) + } + // t.Version (int64) (int64) + case "ver": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Version = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} diff --git a/atproto/labeling/label.go b/atproto/labeling/label.go new file mode 100644 index 000000000..5bdd905a3 --- /dev/null +++ b/atproto/labeling/label.go @@ -0,0 +1,168 @@ +package labeling + +import ( + "bytes" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// version of the label data fromat implemented by this package +const ATPROTO_LABEL_VERSION int64 = 1 + +type Label struct { + CID *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + CreatedAt string `json:"cts" cborgen:"cts"` + ExpiresAt *string `json:"exp,omitempty" cborgen:"exp,omitempty"` + Negated *bool `json:"neg,omitempty" cborgen:"neg,omitempty"` + SourceDID string `json:"src" cborgen:"src"` + URI string `json:"uri" cborgen:"uri"` + Val string `json:"val" cborgen:"val"` + Version int64 `json:"ver" cborgen:"ver"` + Sig atdata.Bytes `json:"sig,omitempty" cborgen:"sig,omitempty"` +} + +// converts to map[string]any for printing as JSON +func (l *Label) Data() map[string]any { + d := map[string]any{ + "cid": l.CID, + "cts": l.CreatedAt, + "src": l.SourceDID, + "uri": l.URI, + "val": l.Val, + "ver": l.Version, + } + if l.CID != nil { + d["cid"] = l.CID + } + if l.ExpiresAt != nil { + d["exp"] = l.ExpiresAt + } + if l.Negated != nil { + d["neg"] = l.Negated + } + if l.Sig != nil { + d["sig"] = atdata.Bytes(l.Sig) + } + return d +} + +// does basic checks on syntax and structure +func (l *Label) VerifySyntax() error { + if l.Version != ATPROTO_LABEL_VERSION { + return fmt.Errorf("unsupported label version: %d", l.Version) + } + if len(l.Val) == 0 { + return fmt.Errorf("empty label value") + } + if l.CID != nil { + _, err := syntax.ParseCID(*l.CID) + if err != nil { + return fmt.Errorf("invalid label: %w", err) + } + } + _, err := syntax.ParseDatetime(l.CreatedAt) + if err != nil { + return fmt.Errorf("invalid label: %w", err) + } + if l.ExpiresAt != nil { + _, err := syntax.ParseDatetime(*l.ExpiresAt) + if err != nil { + return fmt.Errorf("invalid label: %w", err) + } + } + _, err = syntax.ParseDID(l.SourceDID) + if err != nil { + return fmt.Errorf("invalid label: %w", err) + } + _, err = syntax.ParseURI(l.URI) + if err != nil { + return fmt.Errorf("invalid label: %w", err) + } + return nil +} + +func (l *Label) UnsignedBytes() ([]byte, error) { + buf := new(bytes.Buffer) + if l.Sig == nil { + if err := l.MarshalCBOR(buf); err != nil { + return nil, err + } + return buf.Bytes(), nil + } + unsigned := Label{ + CID: l.CID, + CreatedAt: l.CreatedAt, + ExpiresAt: l.ExpiresAt, + Negated: l.Negated, + SourceDID: l.SourceDID, + URI: l.URI, + Val: l.Val, + Version: l.Version, + } + if err := unsigned.MarshalCBOR(buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Signs the commit, storing the signature in the `Sig` field +func (l *Label) Sign(privkey atcrypto.PrivateKey) error { + b, err := l.UnsignedBytes() + if err != nil { + return err + } + sig, err := privkey.HashAndSign(b) + if err != nil { + return err + } + l.Sig = sig + return nil +} + +// Verifies `Sig` field using the provided key. Returns `nil` if signature is valid. +func (l *Label) VerifySignature(pubkey atcrypto.PublicKey) error { + if l.Sig == nil { + return fmt.Errorf("can not verify unsigned commit") + } + b, err := l.UnsignedBytes() + if err != nil { + return err + } + return pubkey.HashAndVerify(b, l.Sig) +} + +func (l *Label) ToLexicon() comatproto.LabelDefs_Label { + return comatproto.LabelDefs_Label{ + Cid: l.CID, + Cts: l.CreatedAt, + Exp: l.ExpiresAt, + Neg: l.Negated, + Sig: []byte(l.Sig), + Src: l.SourceDID, + Uri: l.URI, + Val: l.Val, + Ver: &l.Version, + } +} + +func FromLexicon(l *comatproto.LabelDefs_Label) Label { + var v int64 = 0 + if l.Ver != nil { + v = *l.Ver + } + return Label{ + CID: l.Cid, + CreatedAt: l.Cts, + ExpiresAt: l.Exp, + Negated: l.Neg, + Sig: []byte(l.Sig), + SourceDID: l.Src, + URI: l.Uri, + Val: l.Val, + Version: v, + } +} diff --git a/atproto/labeling/label_test.go b/atproto/labeling/label_test.go new file mode 100644 index 000000000..d90e69246 --- /dev/null +++ b/atproto/labeling/label_test.go @@ -0,0 +1,153 @@ +package labeling + +import ( + "encoding/json" + "testing" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/atcrypto" + + "github.com/stretchr/testify/assert" +) + +func TestVerifyLabel(t *testing.T) { + assert := assert.New(t) + + pubkeyStr := "zQ3shcnfWLQN1bY4d2patsEAYFzy4xp1zdckEvHsV7S4ocTnC" + pubkey, err := atcrypto.ParsePublicMultibase(pubkeyStr) + if err != nil { + t.Fatal(err) + } + + labelJSON := []byte(`{ + "ver": 1, + "src": "did:plc:n3timvoib5nau7gvwd6cshap", + "uri": "did:plc:44ybard66vv44zksje25o7dz", + "val": "bladerunner", + "cts": "2024-10-23T17:51:19.128Z", + "sig": { + "$bytes": "uCRNA5mTzh078T5xZtkvLEt/O+z0gsKM3aqRI/lVAB8ZtbMnznwS/JwHopZE40JhNNDj80z8gsDLAp/hWqG5Pg" + } + }`) + + var l Label + if err := json.Unmarshal(labelJSON, &l); err != nil { + t.Fatal(err) + } + + assert.NoError(l.VerifySyntax()) + assert.NoError(l.VerifySignature(pubkey)) + + // check that signature fails after mutation + l.Val = "wrong" + assert.NoError(l.VerifySyntax()) + assert.Error(l.VerifySignature(pubkey)) + + // version check + l.Version = 0 + assert.Error(l.VerifySyntax()) +} + +func TestParseLabel(t *testing.T) { + assert := assert.New(t) + + unsignedJSON := []byte(`{ + "cts": "2024-10-23T17:51:19.128Z", + "src": "did:plc:n3timvoib5nau7gvwd6cshap", + "uri": "did:plc:44ybard66vv44zksje25o7dz", + "val": "bladerunner", + "ver": 1 + }`) + + var l Label + if err := json.Unmarshal(unsignedJSON, &l); err != nil { + t.Fatal(err) + } + assert.NoError(l.VerifySyntax()) +} + +func TestSignLabel(t *testing.T) { + assert := assert.New(t) + + l := Label{ + Version: ATPROTO_LABEL_VERSION, + CreatedAt: "2024-10-23T17:51:19.128Z", + URI: "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.actor.profile/self", + Val: "good", + SourceDID: "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + } + + priv, err := atcrypto.GeneratePrivateKeyK256() + if err != nil { + t.Fatal(err) + } + + pub, err := priv.PublicKey() + if err != nil { + t.Fatal(err) + } + + assert.NoError(l.Sign(priv)) + assert.NoError(l.VerifySignature(pub)) +} + +func TestToLexicon(t *testing.T) { + assert := assert.New(t) + + expiresAt := "2025-07-28T23:53:19.804Z" + negated := true + cid := "bafyreifxykqhed72s26cr4i64rxvrtofeqrly3j4vjzbkvo3ckkjbxjqtq" + + l := Label{ + CID: &cid, + CreatedAt: "2024-10-23T17:51:19.128Z", + ExpiresAt: &expiresAt, + Negated: &negated, + SourceDID: "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + URI: "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.actor.profile/self", + Val: "good", + Version: ATPROTO_LABEL_VERSION, + Sig: []byte("sig"), // invalid, but we only care about the conversion + } + + lex := l.ToLexicon() + assert.Equal(l.Version, *lex.Ver) + assert.Equal(l.CreatedAt, lex.Cts) + assert.Equal(l.URI, lex.Uri) + assert.Equal(l.Val, lex.Val) + assert.Equal(l.CID, lex.Cid) + assert.Equal(l.ExpiresAt, lex.Exp) + assert.Equal(l.Negated, lex.Neg) + assert.Equal(l.SourceDID, lex.Src) +} + +func TestFromLexicon(t *testing.T) { + assert := assert.New(t) + + expiresAt := "2025-07-28T23:53:19.804Z" + negated := true + cid := "bafyreifxykqhed72s26cr4i64rxvrtofeqrly3j4vjzbkvo3ckkjbxjqtq" + version := int64(1) + + lex := &comatproto.LabelDefs_Label{ + Cid: &cid, + Cts: "2024-10-23T17:51:19.128Z", + Exp: &expiresAt, + Neg: &negated, + Src: "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + Uri: "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.actor.profile/self", + Val: "good", + Ver: &version, + Sig: []byte("sig"), // invalid, but we only care about the conversion + } + + l := FromLexicon(lex) + assert.Equal(lex.Ver, &l.Version) + assert.Equal(lex.Cts, l.CreatedAt) + assert.Equal(lex.Uri, l.URI) + assert.Equal(lex.Val, l.Val) + assert.Equal(lex.Cid, l.CID) + assert.Equal(lex.Exp, l.ExpiresAt) + assert.Equal(lex.Neg, l.Negated) + assert.Equal(lex.Src, l.SourceDID) +} diff --git a/atproto/lexicon/catalog.go b/atproto/lexicon/catalog.go new file mode 100644 index 000000000..ddcf8634c --- /dev/null +++ b/atproto/lexicon/catalog.go @@ -0,0 +1,138 @@ +package lexicon + +import ( + "embed" + "encoding/json" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" +) + +// Interface type for a resolver or container of lexicon schemas, and methods for validating generic data against those schemas. +type Catalog interface { + // Looks up a schema reference (NSID string with optional fragment) to a Schema object. + Resolve(ref string) (*Schema, error) +} + +// Trivial in-memory Lexicon Catalog implementation. +type BaseCatalog struct { + schemas map[string]Schema +} + +// Creates a new empty BaseCatalog +func NewBaseCatalog() BaseCatalog { + return BaseCatalog{ + schemas: make(map[string]Schema), + } +} + +// Returns a scheman definition (`Schema` struct) for a Lexicon reference. +// +// A Lexicon ref string is an NSID with an optional #-separated fragment. If the fragment isn't specified, '#main' is used by default. +func (c *BaseCatalog) Resolve(ref string) (*Schema, error) { + if ref == "" { + return nil, fmt.Errorf("tried to resolve empty string name") + } + // default to #main if name doesn't have a fragment + if !strings.Contains(ref, "#") { + ref = ref + "#main" + } + s, ok := c.schemas[ref] + if !ok { + return nil, fmt.Errorf("schema not found in catalog: %s", ref) + } + return &s, nil +} + +// Inserts a schema loaded from a JSON file in to the catalog. +func (c *BaseCatalog) AddSchemaFile(sf SchemaFile) error { + + if err := sf.FinishParse(); err != nil { + return err + } + + if err := sf.CheckSchema(); err != nil { + return err + } + + base := sf.ID + for frag, def := range sf.Defs { + name := base + "#" + frag + if _, ok := c.schemas[name]; ok { + return fmt.Errorf("catalog already contained a schema with name: %s", name) + } + s := Schema{ + ID: name, + Def: def.Inner, + } + c.schemas[name] = s + } + return nil +} + +// internal helper for loading JSON files from bytes +func (c *BaseCatalog) addSchemaFromBytes(b []byte) error { + var sf SchemaFile + if err := json.Unmarshal(b, &sf); err != nil { + return err + } + if err := c.AddSchemaFile(sf); err != nil { + return err + } + return nil +} + +// Recursively loads all '.json' files from a directory in to the catalog. +func (c *BaseCatalog) LoadDirectory(dirPath string) error { + walkFunc := func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if !strings.HasSuffix(p, ".json") { + return nil + } + slog.Debug("loading Lexicon schema file", "path", p) + f, err := os.Open(p) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + b, err := io.ReadAll(f) + if err != nil { + return err + } + return c.addSchemaFromBytes(b) + } + return filepath.WalkDir(dirPath, walkFunc) +} + +// Recursively loads all '.json' files from an embed.FS +func (c *BaseCatalog) LoadEmbedFS(efs embed.FS) error { + walkFunc := func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if !strings.HasSuffix(p, ".json") { + return nil + } + + slog.Debug("loading embedded Lexicon schema file", "path", p) + b, err := efs.ReadFile(p) + if err != nil { + return err + } + return c.addSchemaFromBytes(b) + } + return fs.WalkDir(efs, ".", walkFunc) +} diff --git a/atproto/lexicon/catalog_test.go b/atproto/lexicon/catalog_test.go new file mode 100644 index 000000000..b2420306a --- /dev/null +++ b/atproto/lexicon/catalog_test.go @@ -0,0 +1,41 @@ +package lexicon + +import ( + "embed" + "testing" + + "github.com/stretchr/testify/assert" +) + +//go:embed testdata/catalog +var embedDir embed.FS + +func TestEmbedCatalog(t *testing.T) { + assert := assert.New(t) + + cat := NewBaseCatalog() + + err := cat.LoadEmbedFS(embedDir) + assert.NoError(err) + + _, err = cat.Resolve("example.lexicon.query") + assert.NoError(err) + + _, err = cat.Resolve("example.lexicon.notThere") + assert.Error(err) +} + +func TestDirCatalog(t *testing.T) { + assert := assert.New(t) + + cat := NewBaseCatalog() + + err := cat.LoadDirectory("testdata/catalog") + assert.NoError(err) + + _, err = cat.Resolve("example.lexicon.query") + assert.NoError(err) + + _, err = cat.Resolve("example.lexicon.notThere") + assert.Error(err) +} diff --git a/atproto/lexicon/cmd/lextool/main.go b/atproto/lexicon/cmd/lextool/main.go new file mode 100644 index 000000000..d63186c1c --- /dev/null +++ b/atproto/lexicon/cmd/lextool/main.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "os" + + "github.com/bluesky-social/indigo/atproto/lexicon" + + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "lex-tool", + Usage: "informal debugging CLI tool for atproto lexicons", + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "parse-schema", + Usage: "parse an individual lexicon schema file (JSON)", + Action: runParseSchema, + }, + &cli.Command{ + Name: "load-directory", + Usage: "try recursively loading all the schemas from a directory", + Action: runLoadDirectory, + }, + &cli.Command{ + Name: "validate-record", + Usage: "fetch from network, validate against catalog", + Action: runValidateRecord, + }, + &cli.Command{ + Name: "resolve", + Usage: "resolves an NSID to a lexicon schema", + Action: runResolve, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + if err := app.Run(context.Background(), os.Args); err != nil { + slog.Error("command failed", "error", err) + os.Exit(-1) + } +} + +func runParseSchema(ctx context.Context, cmd *cli.Command) error { + p := cmd.Args().First() + if p == "" { + return fmt.Errorf("need to provide path to a schema file as an argument") + } + + f, err := os.Open(p) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + b, err := io.ReadAll(f) + if err != nil { + return err + } + + var sf lexicon.SchemaFile + if err := json.Unmarshal(b, &sf); err != nil { + return err + } + out, err := json.MarshalIndent(sf, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} + +func runLoadDirectory(ctx context.Context, cmd *cli.Command) error { + p := cmd.Args().First() + if p == "" { + return fmt.Errorf("need to provide directory path as an argument") + } + + c := lexicon.NewBaseCatalog() + err := c.LoadDirectory(p) + if err != nil { + return err + } + + fmt.Println("success!") + return nil +} + +func runResolve(ctx context.Context, cmd *cli.Command) error { + ref := cmd.Args().First() + if ref == "" { + return fmt.Errorf("need to provide NSID as an argument") + } + + c := lexicon.NewResolvingCatalog() + schema, err := c.Resolve(ref) + if err != nil { + return err + } + + out, err := json.MarshalIndent(schema, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} diff --git a/atproto/lexicon/cmd/lextool/net.go b/atproto/lexicon/cmd/lextool/net.go new file mode 100644 index 000000000..6b6ea8ac9 --- /dev/null +++ b/atproto/lexicon/cmd/lextool/net.go @@ -0,0 +1,82 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/urfave/cli/v3" +) + +func runValidateRecord(ctx context.Context, cmd *cli.Command) error { + args := cmd.Args().Slice() + if len(args) != 2 { + return fmt.Errorf("expected two args (catalog path and AT-URI)") + } + p := args[0] + if p == "" { + return fmt.Errorf("need to provide directory path as an argument") + } + + cat := lexicon.NewBaseCatalog() + err := cat.LoadDirectory(p) + if err != nil { + return err + } + + aturi, err := syntax.ParseATURI(args[1]) + if err != nil { + return err + } + if aturi.RecordKey() == "" { + return fmt.Errorf("need a full, not partial, AT-URI: %s", aturi) + } + dir := identity.DefaultDirectory() + ident, err := dir.Lookup(ctx, aturi.Authority()) + if err != nil { + return fmt.Errorf("resolving AT-URI authority: %v", err) + } + pdsURL := ident.PDSEndpoint() + if pdsURL == "" { + return fmt.Errorf("could not resolve PDS endpoint for AT-URI account: %s", ident.DID.String()) + } + + slog.Info("fetching record", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String()) + url := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s", + pdsURL, ident.DID, aturi.Collection(), aturi.RecordKey()) + resp, err := http.Get(url) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("fetch failed") + } + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + body, err := atdata.UnmarshalJSON(respBytes) + if err != nil { + return err + } + record, ok := body["value"].(map[string]any) + if !ok { + return fmt.Errorf("fetched record was not an object") + } + + slog.Info("validating", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String()) + err = lexicon.ValidateRecord(&cat, record, aturi.Collection().String(), lexicon.LenientMode) + if err != nil { + return err + } + fmt.Println("success!") + return nil +} diff --git a/atproto/lexicon/docs.go b/atproto/lexicon/docs.go new file mode 100644 index 000000000..754179933 --- /dev/null +++ b/atproto/lexicon/docs.go @@ -0,0 +1,4 @@ +/* +Package atproto/lexicon provides generic Lexicon schema parsing and run-time validation. +*/ +package lexicon diff --git a/atproto/lexicon/examples_test.go b/atproto/lexicon/examples_test.go new file mode 100644 index 000000000..150700ff6 --- /dev/null +++ b/atproto/lexicon/examples_test.go @@ -0,0 +1,41 @@ +package lexicon + +import ( + "fmt" + + "github.com/bluesky-social/indigo/atproto/atdata" +) + +func ExampleValidateRecord() { + + // First load Lexicon schema JSON files from local disk. + cat := NewBaseCatalog() + if err := cat.LoadDirectory("testdata/catalog"); err != nil { + panic("failed to load lexicons") + } + + // Parse record JSON data using atdata helper + recordJSON := `{ + "$type": "example.lexicon.record", + "integer": 123, + "formats": { + "did": "did:web:example.com", + "aturi": "at://handle.example.com/com.example.nsid/asdf123", + "datetime": "2023-10-30T22:25:23Z", + "language": "en", + "tid": "3kznmn7xqxl22" + } + }` + + recordData, err := atdata.UnmarshalJSON([]byte(recordJSON)) + if err != nil { + panic("failed to parse record JSON") + } + + if err := ValidateRecord(&cat, recordData, "example.lexicon.record", 0); err != nil { + fmt.Printf("Schema validation failed: %v\n", err) + } else { + fmt.Println("Success!") + } + // Output: Success! +} diff --git a/atproto/lexicon/extract_type.go b/atproto/lexicon/extract_type.go new file mode 100644 index 000000000..51d09ba93 --- /dev/null +++ b/atproto/lexicon/extract_type.go @@ -0,0 +1,20 @@ +package lexicon + +import ( + "encoding/json" +) + +// Helper type for extracting record type from JSON +type genericSchemaDef struct { + Type string `json:"type"` +} + +// Parses the top-level $type field from generic atproto JSON data +func ExtractTypeJSON(b []byte) (string, error) { + var gsd genericSchemaDef + if err := json.Unmarshal(b, &gsd); err != nil { + return "", err + } + + return gsd.Type, nil +} diff --git a/atproto/lexicon/interop_language_test.go b/atproto/lexicon/interop_language_test.go new file mode 100644 index 000000000..43a4396e9 --- /dev/null +++ b/atproto/lexicon/interop_language_test.go @@ -0,0 +1,105 @@ +package lexicon + +import ( + "encoding/json" + "fmt" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +type LexiconFixture struct { + Name string `json:"name"` + Lexicon json.RawMessage `json:"lexicon"` +} + +func TestInteropLexiconValid(t *testing.T) { + + f, err := os.Open("testdata/lexicon-valid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []LexiconFixture + if err := json.Unmarshal(jsonBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, f := range fixtures { + testLexiconFixtureValid(t, f) + } +} + +func testLexiconFixtureValid(t *testing.T, fixture LexiconFixture) { + assert := assert.New(t) + + var schema SchemaFile + if err := json.Unmarshal(fixture.Lexicon, &schema); err != nil { + t.Fatal(err) + } + + outBytes, err := json.Marshal(schema) + if err != nil { + t.Fatal(err) + } + + var beforeMap map[string]any + if err := json.Unmarshal(fixture.Lexicon, &beforeMap); err != nil { + t.Fatal(err) + } + + var afterMap map[string]any + if err := json.Unmarshal(outBytes, &afterMap); err != nil { + t.Fatal(err) + } + + assert.Equal(beforeMap, afterMap) +} + +func TestInteropLexiconInvalid(t *testing.T) { + + f, err := os.Open("testdata/lexicon-invalid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []LexiconFixture + if err := json.Unmarshal(jsonBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, f := range fixtures { + testLexiconFixtureInvalid(t, f) + } +} + +func testLexiconFixtureInvalid(t *testing.T, fixture LexiconFixture) { + assert := assert.New(t) + + var schema SchemaFile + err := json.Unmarshal(fixture.Lexicon, &schema) + if err == nil { + err = schema.FinishParse() + } + if err == nil { + err = schema.CheckSchema() + } + if err == nil { + fmt.Println(fixture.Name) + } + assert.Error(err) +} diff --git a/atproto/lexicon/interop_record_test.go b/atproto/lexicon/interop_record_test.go new file mode 100644 index 000000000..bfe1611e2 --- /dev/null +++ b/atproto/lexicon/interop_record_test.go @@ -0,0 +1,92 @@ +package lexicon + +import ( + "encoding/json" + "fmt" + "io" + "os" + "testing" + + "github.com/bluesky-social/indigo/atproto/atdata" + + "github.com/stretchr/testify/assert" +) + +type RecordFixture struct { + Name string `json:"name"` + RecordKey string `json:"rkey"` + Data json.RawMessage `json:"data"` +} + +func TestInteropRecordValid(t *testing.T) { + assert := assert.New(t) + + cat := NewBaseCatalog() + if err := cat.LoadDirectory("testdata/catalog"); err != nil { + t.Fatal(err) + } + + f, err := os.Open("testdata/record-data-valid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []RecordFixture + if err := json.Unmarshal(jsonBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, fixture := range fixtures { + fmt.Println(fixture.Name) + d, err := atdata.UnmarshalJSON(fixture.Data) + if err != nil { + t.Fatal(err) + } + + assert.NoError(ValidateRecord(&cat, d, "example.lexicon.record", 0)) + } +} + +func TestInteropRecordInvalid(t *testing.T) { + assert := assert.New(t) + + cat := NewBaseCatalog() + if err := cat.LoadDirectory("testdata/catalog"); err != nil { + t.Fatal(err) + } + + f, err := os.Open("testdata/record-data-invalid.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var fixtures []RecordFixture + if err := json.Unmarshal(jsonBytes, &fixtures); err != nil { + t.Fatal(err) + } + + for _, fixture := range fixtures { + fmt.Println(fixture.Name) + d, err := atdata.UnmarshalJSON(fixture.Data) + if err != nil { + t.Fatal(err) + } + err = ValidateRecord(&cat, d, "example.lexicon.record", 0) + if err == nil { + fmt.Println(" FAIL") + } + assert.Error(err) + } +} diff --git a/atproto/lexicon/language.go b/atproto/lexicon/language.go new file mode 100644 index 000000000..8748a8af7 --- /dev/null +++ b/atproto/lexicon/language.go @@ -0,0 +1,1114 @@ +package lexicon + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/rivo/uniseg" +) + +// enum type to represent any of the schema fields +type SchemaDef struct { + Inner any +} + +// Checks that the schema definition itself is valid (recursively). +func (s *SchemaDef) CheckSchema() error { + switch v := s.Inner.(type) { + case SchemaRecord: + return v.CheckSchema() + case SchemaQuery: + return v.CheckSchema() + case SchemaProcedure: + return v.CheckSchema() + case SchemaSubscription: + return v.CheckSchema() + case SchemaPermissionSet: + return v.CheckSchema() + case SchemaPermission: + return v.CheckSchema() + case SchemaBoolean: + return v.CheckSchema() + case SchemaInteger: + return v.CheckSchema() + case SchemaString: + return v.CheckSchema() + case SchemaBytes: + return v.CheckSchema() + case SchemaCIDLink: + return v.CheckSchema() + case SchemaArray: + return v.CheckSchema() + case SchemaObject: + return v.CheckSchema() + case SchemaBlob: + return v.CheckSchema() + case SchemaParams: + return v.CheckSchema() + case SchemaToken: + return v.CheckSchema() + case SchemaRef: + return v.CheckSchema() + case SchemaUnion: + return v.CheckSchema() + case SchemaUnknown: + return v.CheckSchema() + default: + return fmt.Errorf("unhandled schema type: %v", reflect.TypeOf(v)) + } +} + +// Checks if def is of "parimary" type +func (s *SchemaDef) IsPrimary() bool { + switch s.Inner.(type) { + case SchemaRecord, SchemaQuery, SchemaProcedure, SchemaSubscription, SchemaPermissionSet: + return true + } + return false +} + +func (s *SchemaDef) IsConcrete() bool { + switch s.Inner.(type) { + case SchemaBoolean, SchemaInteger, SchemaString, SchemaBytes, SchemaCIDLink, SchemaBlob: + return true + } + return false +} + +// Checks if type can be a top-level "named" definition +func (s *SchemaDef) IsDefinable() bool { + switch s.Inner.(type) { + case SchemaUnknown, SchemaUnion, SchemaRef, SchemaParams, SchemaPermission: + return false + } + return true +} + +// Checks if the type is allowed to be included in object fields. +func (s *SchemaDef) IsFieldType() bool { + if s.IsConcrete() { + return true + } + switch s.Inner.(type) { + case SchemaArray, SchemaObject, SchemaRef, SchemaUnion, SchemaUnknown: + return true + } + return false +} + +// Helper to recurse down the definition tree and set full references on any sub-schemas which need to embed that metadata +func (s *SchemaDef) setBase(base string) { + switch v := s.Inner.(type) { + case SchemaRecord: + for i, val := range v.Record.Properties { + val.setBase(base) + v.Record.Properties[i] = val + } + s.Inner = v + case SchemaQuery: + if v.Parameters != nil { + for i, val := range v.Parameters.Properties { + val.setBase(base) + v.Parameters.Properties[i] = val + } + } + if v.Output != nil && v.Output.Schema != nil { + v.Output.Schema.setBase(base) + } + s.Inner = v + case SchemaProcedure: + if v.Parameters != nil { + for i, val := range v.Parameters.Properties { + val.setBase(base) + v.Parameters.Properties[i] = val + } + } + if v.Input != nil && v.Input.Schema != nil { + v.Input.Schema.setBase(base) + } + if v.Output != nil && v.Output.Schema != nil { + v.Output.Schema.setBase(base) + } + s.Inner = v + case SchemaSubscription: + if v.Parameters != nil { + for i, val := range v.Parameters.Properties { + val.setBase(base) + v.Parameters.Properties[i] = val + } + } + u := SchemaDef{Inner: v.Message.Schema} + u.setBase(base) + i := u.Inner.(SchemaUnion) + v.Message.Schema = i + s.Inner = v + case SchemaArray: + v.Items.setBase(base) + s.Inner = v + case SchemaObject: + for i, val := range v.Properties { + val.setBase(base) + v.Properties[i] = val + } + s.Inner = v + case SchemaParams: + for i, val := range v.Properties { + val.setBase(base) + v.Properties[i] = val + } + s.Inner = v + case SchemaRef: + // add fully-qualified name + if strings.HasPrefix(v.Ref, "#") { + v.fullRef = base + v.Ref + } else { + v.fullRef = v.Ref + } + s.Inner = v + case SchemaUnion: + // add fully-qualified name + for _, ref := range v.Refs { + if strings.HasPrefix(ref, "#") { + ref = base + ref + } + v.fullRefs = append(v.fullRefs, ref) + } + s.Inner = v + } + return +} + +func (s SchemaDef) MarshalJSON() ([]byte, error) { + return json.Marshal(s.Inner) +} + +func (s *SchemaDef) UnmarshalJSON(b []byte) error { + t, err := ExtractTypeJSON(b) + if err != nil { + return err + } + // TODO: should we call CheckSchema here, instead of in lexicon loading? + switch t { + case "record": + v := new(SchemaRecord) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "query": + v := new(SchemaQuery) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "procedure": + v := new(SchemaProcedure) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "subscription": + v := new(SchemaSubscription) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "permission-set": + v := new(SchemaPermissionSet) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "permission": + v := new(SchemaPermission) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "boolean": + v := new(SchemaBoolean) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "integer": + v := new(SchemaInteger) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "string": + v := new(SchemaString) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "bytes": + v := new(SchemaBytes) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "cid-link": + v := new(SchemaCIDLink) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "array": + v := new(SchemaArray) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "object": + v := new(SchemaObject) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "blob": + v := new(SchemaBlob) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "params": + v := new(SchemaParams) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "token": + v := new(SchemaToken) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "ref": + v := new(SchemaRef) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "union": + v := new(SchemaUnion) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + case "unknown": + v := new(SchemaUnknown) + if err = json.Unmarshal(b, v); err != nil { + return err + } + s.Inner = *v + return nil + default: + return fmt.Errorf("unexpected schema type: %s", t) + } +} + +type SchemaRecord struct { + Type string `json:"type"` // "record" + Description *string `json:"description,omitempty"` + Key string `json:"key"` + Record SchemaObject `json:"record"` +} + +func (s *SchemaRecord) CheckSchema() error { + if s.Type != "record" { + return fmt.Errorf("expected 'record' schema") + } + switch s.Key { + case "": + return fmt.Errorf("record key specifier is required") + case "tid", "nsid", "any": + // pass + default: + if !strings.HasPrefix(s.Key, "literal:") { + return fmt.Errorf("invalid record key specifier: %s", s.Key) + } + } + return s.Record.CheckSchema() +} + +type SchemaQuery struct { + Type string `json:"type"` // "query" + Description *string `json:"description,omitempty"` + Parameters *SchemaParams `json:"parameters"` // optional + Output *SchemaBody `json:"output"` // optional + Errors []SchemaError `json:"errors,omitempty"` // optional +} + +func (s *SchemaQuery) CheckSchema() error { + if s.Type != "query" { + return fmt.Errorf("expected 'query' schema") + } + if s.Output != nil { + if err := s.Output.CheckSchema(); err != nil { + return err + } + } + if s.Parameters != nil { + if err := s.Parameters.CheckSchema(); err != nil { + return err + } + } + for _, e := range s.Errors { + if err := e.CheckSchema(); err != nil { + return err + } + } + return nil +} + +type SchemaProcedure struct { + Type string `json:"type"` // "procedure" + Description *string `json:"description,omitempty"` + Parameters *SchemaParams `json:"parameters"` // optional + Output *SchemaBody `json:"output"` // optional + Errors []SchemaError `json:"errors,omitempty"` // optional + Input *SchemaBody `json:"input"` // optional +} + +func (s *SchemaProcedure) CheckSchema() error { + if s.Type != "procedure" { + return fmt.Errorf("expected 'procedure' schema") + } + if s.Input != nil { + if err := s.Input.CheckSchema(); err != nil { + return err + } + } + if s.Output != nil { + if err := s.Output.CheckSchema(); err != nil { + return err + } + } + if s.Parameters != nil { + if err := s.Parameters.CheckSchema(); err != nil { + return err + } + } + for _, e := range s.Errors { + if err := e.CheckSchema(); err != nil { + return err + } + } + return nil +} + +type SchemaSubscription struct { + Type string `json:"type"` // "subscription" + Description *string `json:"description,omitempty"` + Parameters *SchemaParams `json:"parameters"` // optional + Message SchemaMessage `json:"message"` +} + +func (s *SchemaSubscription) CheckSchema() error { + if s.Type != "subscription" { + return fmt.Errorf("expected 'subscription' schema") + } + if err := s.Message.CheckSchema(); err != nil { + return err + } + if s.Parameters != nil { + if err := s.Parameters.CheckSchema(); err != nil { + return err + } + } + return nil +} + +type SchemaPermissionSet struct { + Type string `json:"type"` // "permission-set" + Description *string `json:"description,omitempty"` + Title *string `json:"title,omitempty"` + TitleLangs map[string]string `json:"title:langs,omitempty"` + Detail *string `json:"detail,omitempty"` + DetailLangs map[string]string `json:"detail:langs,omitempty"` + Permissions []SchemaPermission `json:"permissions"` +} + +func (s *SchemaPermissionSet) CheckSchema() error { + if s.Type != "permission-set" { + return fmt.Errorf("expected 'permission-set' schema") + } + for lang, _ := range s.TitleLangs { + _, err := syntax.ParseLanguage(lang) + if err != nil { + return err + } + } + for lang, _ := range s.DetailLangs { + _, err := syntax.ParseLanguage(lang) + if err != nil { + return err + } + } + for _, p := range s.Permissions { + if err := p.CheckSchema(); err != nil { + return err + } + } + return nil +} + +type SchemaPermission struct { + Type string `json:"type"` // "permission" + Description *string `json:"description,omitempty"` + + Resource string `json:"resource"` + Collection []string `json:"collection,omitempty"` + Action []string `json:"action,omitempty"` + LXM []string `json:"lxm,omitempty"` + Audience string `json:"aud,omitempty"` + InheritAud bool `json:"inheritAud,omitempty"` +} + +func (s *SchemaPermission) CheckSchema() error { + if s.Type != "permission" { + return fmt.Errorf("expected 'permission' schema") + } + switch s.Resource { + case "repo": + if len(s.Collection) == 0 { + return fmt.Errorf("repo permission requires 'collection'") + } + for _, coll := range s.Collection { + if coll == "*" { + return fmt.Errorf("repo permission does not support wildcard when in permission set") + } + _, err := syntax.ParseNSID(coll) + if err != nil { + return fmt.Errorf("repo permission: %w", err) + } + } + for _, act := range s.Action { + if act != "create" && act != "update" && act != "delete" { + return fmt.Errorf("repo permission unsupported action: %s", act) + } + } + case "rpc": + if len(s.LXM) == 0 { + return fmt.Errorf("rpc permission requires 'lxm'") + } + for _, lxm := range s.LXM { + if lxm == "*" { + if s.Audience == "*" { + // TODO: is this necessary here? + return fmt.Errorf("rpc permission can't have both 'lxm' and 'aud' be '*'") + } + continue + } + _, err := syntax.ParseNSID(lxm) + if err != nil { + return fmt.Errorf("rpc permission: %w", err) + } + } + if (s.InheritAud == true && s.Audience != "") || (s.InheritAud == false && s.Audience == "") { + return fmt.Errorf("rpc permission must have eith 'aud' or 'inheritAud' defined") + } + if s.Audience != "" && s.Audience != "*" { + return fmt.Errorf("rpc permission 'aud' can't have service DID in permission set") + } + case "blob", "account", "identity": + return fmt.Errorf("%s permission not allowed in permission sets", s.Resource) + default: + return fmt.Errorf("unsupported permission resource: %s", s.Resource) + } + return nil +} + +type SchemaBody struct { + Description *string `json:"description,omitempty"` + Encoding string `json:"encoding"` // required, mimetype + Schema *SchemaDef `json:"schema"` // optional; type:object, type:ref, or type:union +} + +func (s *SchemaBody) CheckSchema() error { + // TODO: any validation of encoding? + if s.Schema != nil { + switch s.Schema.Inner.(type) { + case SchemaObject, SchemaRef, SchemaUnion: + // pass + default: + return fmt.Errorf("body type can only have object, ref, or union schema") + } + if err := s.Schema.CheckSchema(); err != nil { + return err + } + } + return nil +} + +type SchemaMessage struct { + Description *string `json:"description,omitempty"` + Schema SchemaUnion `json:"schema"` // required; type:union only +} + +func (s *SchemaMessage) CheckSchema() error { + return s.Schema.CheckSchema() +} + +type SchemaError struct { + Name string `json:"name"` + Description *string `json:"description"` +} + +func (s *SchemaError) CheckSchema() error { + return nil +} +func (s *SchemaError) Validate(d any) error { + e, ok := d.(map[string]any) + if !ok { + return fmt.Errorf("expected an object in error position") + } + n, ok := e["error"] + if !ok { + return fmt.Errorf("expected error type") + } + if n != s.Name { + return fmt.Errorf("error type mis-match: %s", n) + } + return nil +} + +type SchemaBoolean struct { + Type string `json:"type"` // "boolean" + Description *string `json:"description,omitempty"` + Default *bool `json:"default,omitempty"` + Const *bool `json:"const,omitempty"` +} + +func (s *SchemaBoolean) CheckSchema() error { + if s.Type != "boolean" { + return fmt.Errorf("expected 'boolean' schema") + } + if s.Default != nil && s.Const != nil { + return fmt.Errorf("schema can't have both 'default' and 'const'") + } + return nil +} + +func (s *SchemaBoolean) Validate(d any) error { + v, ok := d.(bool) + if !ok { + return fmt.Errorf("expected a boolean") + } + if s.Const != nil && v != *s.Const { + return fmt.Errorf("boolean val didn't match constant (%v): %v", *s.Const, v) + } + return nil +} + +type SchemaInteger struct { + Type string `json:"type"` // "integer" + Description *string `json:"description,omitempty"` + Minimum *int `json:"minimum,omitempty"` + Maximum *int `json:"maximum,omitempty"` + Enum []int `json:"enum,omitempty"` + Default *int `json:"default,omitempty"` + Const *int `json:"const,omitempty"` +} + +func (s *SchemaInteger) CheckSchema() error { + if s.Type != "integer" { + return fmt.Errorf("expected 'integer' schema") + } + // TODO: enforce min/max against enum, default, const + if s.Default != nil && s.Const != nil { + return fmt.Errorf("schema can't have both 'default' and 'const'") + } + if s.Minimum != nil && s.Maximum != nil && *s.Maximum < *s.Minimum { + return fmt.Errorf("schema max < min") + } + return nil +} + +func (s *SchemaInteger) Validate(d any) error { + v64, ok := d.(int64) + if !ok { + return fmt.Errorf("expected an integer") + } + v := int(v64) + if s.Const != nil && v != *s.Const { + return fmt.Errorf("integer val didn't match constant (%d): %d", *s.Const, v) + } + if (s.Minimum != nil && v < *s.Minimum) || (s.Maximum != nil && v > *s.Maximum) { + return fmt.Errorf("integer val outside specified range: %d", v) + } + if len(s.Enum) != 0 { + inEnum := false + for _, e := range s.Enum { + if e == v { + inEnum = true + break + } + } + if !inEnum { + return fmt.Errorf("integer val not in required enum: %d", v) + } + } + return nil +} + +type SchemaString struct { + Type string `json:"type"` // "string" + Description *string `json:"description,omitempty"` + Format *string `json:"format,omitempty"` + MinLength *int `json:"minLength,omitempty"` + MaxLength *int `json:"maxLength,omitempty"` + MinGraphemes *int `json:"minGraphemes,omitempty"` + MaxGraphemes *int `json:"maxGraphemes,omitempty"` + KnownValues []string `json:"knownValues,omitempty"` + Enum []string `json:"enum,omitempty"` + Default *string `json:"default,omitempty"` + Const *string `json:"const,omitempty"` +} + +func (s *SchemaString) CheckSchema() error { + if s.Type != "string" { + return fmt.Errorf("expected 'string' schema") + } + // TODO: enforce min/max against enum, default, const + if s.Default != nil && s.Const != nil { + return fmt.Errorf("schema can't have both 'default' and 'const'") + } + if s.MinLength != nil && s.MaxLength != nil && *s.MaxLength < *s.MinLength { + return fmt.Errorf("schema max < min") + } + if s.MinGraphemes != nil && s.MaxGraphemes != nil && *s.MaxGraphemes < *s.MinGraphemes { + return fmt.Errorf("schema max < min") + } + if (s.MinLength != nil && *s.MinLength < 0) || + (s.MaxLength != nil && *s.MaxLength < 0) || + (s.MinGraphemes != nil && *s.MinGraphemes < 0) || + (s.MaxGraphemes != nil && *s.MaxGraphemes < 0) { + return fmt.Errorf("string schema min or max below zero") + } + if s.Format != nil { + switch *s.Format { + case "at-identifier", "at-uri", "cid", "datetime", "did", "handle", "nsid", "uri", "language", "tid", "record-key": + // pass + default: + return fmt.Errorf("unknown string format: %s", *s.Format) + } + } + return nil +} + +func (s *SchemaString) Validate(d any, flags ValidateFlags) error { + v, ok := d.(string) + if !ok { + return fmt.Errorf("expected a string: %v", reflect.TypeOf(d)) + } + if s.Const != nil && v != *s.Const { + return fmt.Errorf("string val didn't match constant (%s): %s", *s.Const, v) + } + // TODO: is this actually counting UTF-8 length? + if (s.MinLength != nil && len(v) < *s.MinLength) || (s.MaxLength != nil && len(v) > *s.MaxLength) { + return fmt.Errorf("string length outside specified range: %d", len(v)) + } + if len(s.Enum) != 0 { + inEnum := false + for _, e := range s.Enum { + if e == v { + inEnum = true + break + } + } + if !inEnum { + return fmt.Errorf("string val not in required enum: %s", v) + } + } + if s.MinGraphemes != nil || s.MaxGraphemes != nil { + lenG := uniseg.GraphemeClusterCount(v) + if (s.MinGraphemes != nil && lenG < *s.MinGraphemes) || (s.MaxGraphemes != nil && lenG > *s.MaxGraphemes) { + return fmt.Errorf("string length (graphemes) outside specified range: %d", lenG) + } + } + if s.Format != nil { + switch *s.Format { + case "at-identifier": + if _, err := syntax.ParseAtIdentifier(v); err != nil { + return err + } + case "at-uri": + if _, err := syntax.ParseATURI(v); err != nil { + return err + } + case "cid": + if _, err := syntax.ParseCID(v); err != nil { + return err + } + case "datetime": + if flags&AllowLenientDatetime != 0 { + if _, err := syntax.ParseDatetimeLenient(v); err != nil { + return err + } + } else { + if _, err := syntax.ParseDatetime(v); err != nil { + return err + } + } + case "did": + if _, err := syntax.ParseDID(v); err != nil { + return err + } + case "handle": + if _, err := syntax.ParseHandle(v); err != nil { + return err + } + case "nsid": + if _, err := syntax.ParseNSID(v); err != nil { + return err + } + case "uri": + if _, err := syntax.ParseURI(v); err != nil { + return err + } + case "language": + if _, err := syntax.ParseLanguage(v); err != nil { + return err + } + case "tid": + if _, err := syntax.ParseTID(v); err != nil { + return err + } + case "record-key": + if _, err := syntax.ParseRecordKey(v); err != nil { + return err + } + } + } + return nil +} + +type SchemaBytes struct { + Type string `json:"type"` // "bytes" + Description *string `json:"description,omitempty"` + MinLength *int `json:"minLength,omitempty"` + MaxLength *int `json:"maxLength,omitempty"` +} + +func (s *SchemaBytes) CheckSchema() error { + if s.Type != "bytes" { + return fmt.Errorf("expected 'bytes' schema") + } + if s.MinLength != nil && s.MaxLength != nil && *s.MaxLength < *s.MinLength { + return fmt.Errorf("schema max < min") + } + if (s.MinLength != nil && *s.MinLength < 0) || + (s.MaxLength != nil && *s.MaxLength < 0) { + return fmt.Errorf("bytes schema min or max below zero") + } + return nil +} + +func (s *SchemaBytes) Validate(d any) error { + v, ok := d.(atdata.Bytes) + if !ok { + return fmt.Errorf("expecting bytes") + } + if (s.MinLength != nil && len(v) < *s.MinLength) || (s.MaxLength != nil && len(v) > *s.MaxLength) { + return fmt.Errorf("bytes size out of bounds: %d", len(v)) + } + return nil +} + +type SchemaCIDLink struct { + Type string `json:"type"` // "cid-link" + Description *string `json:"description,omitempty"` +} + +func (s *SchemaCIDLink) CheckSchema() error { + if s.Type != "cid-link" { + return fmt.Errorf("expected 'cid-link' schema") + } + return nil +} + +func (s *SchemaCIDLink) Validate(d any) error { + _, ok := d.(atdata.CIDLink) + if !ok { + return fmt.Errorf("expecting a cid-link") + } + return nil +} + +type SchemaArray struct { + Type string `json:"type"` // "array" + Description *string `json:"description,omitempty"` + Items SchemaDef `json:"items"` + MinLength *int `json:"minLength,omitempty"` + MaxLength *int `json:"maxLength,omitempty"` +} + +func (s *SchemaArray) CheckSchema() error { + if s.Type != "array" { + return fmt.Errorf("expected 'array' schema") + } + if s.MinLength != nil && s.MaxLength != nil && *s.MaxLength < *s.MinLength { + return fmt.Errorf("schema max < min") + } + if (s.MinLength != nil && *s.MinLength < 0) || + (s.MaxLength != nil && *s.MaxLength < 0) { + return fmt.Errorf("array schema min or max below zero") + } + if !s.Items.IsFieldType() { + return fmt.Errorf("array schema elements not allowed type") + } + return s.Items.CheckSchema() +} + +type SchemaObject struct { + Type string `json:"type"` // "object" + Description *string `json:"description,omitempty"` + Properties map[string]SchemaDef `json:"properties"` + Required []string `json:"required,omitempty"` + Nullable []string `json:"nullable,omitempty"` +} + +func (s *SchemaObject) CheckSchema() error { + if s.Type != "object" { + return fmt.Errorf("expected 'object' schema") + } + // TODO: check for set intersection between required and nullable + // TODO: check for set uniqueness of required and nullable + for _, k := range s.Required { + if _, ok := s.Properties[k]; !ok { + return fmt.Errorf("object 'required' field not in properties: %s", k) + } + } + for _, k := range s.Nullable { + if _, ok := s.Properties[k]; !ok { + return fmt.Errorf("object 'nullable' field not in properties: %s", k) + } + } + for k, def := range s.Properties { + // TODO: more checks on field name? + if len(k) == 0 { + return fmt.Errorf("empty object schema field name not allowed") + } + if !def.IsFieldType() { + return fmt.Errorf("object schema property not an allowed type") + } + if err := def.CheckSchema(); err != nil { + return err + } + } + return nil +} + +// Checks if a field name 'k' is one of the Nullable fields for this object +func (s *SchemaObject) IsNullable(k string) bool { + for _, el := range s.Nullable { + if el == k { + return true + } + } + return false +} + +type SchemaBlob struct { + Type string `json:"type"` // "blob" + Description *string `json:"description,omitempty"` + Accept []string `json:"accept,omitempty"` + MaxSize *int `json:"maxSize,omitempty"` +} + +func (s *SchemaBlob) CheckSchema() error { + if s.Type != "blob" { + return fmt.Errorf("expected 'blob' schema") + } + // TODO: validate Accept (mimetypes)? + if s.MaxSize != nil && *s.MaxSize <= 0 { + return fmt.Errorf("blob max size less or equal to zero") + } + return nil +} + +func (s *SchemaBlob) Validate(d any, flags ValidateFlags) error { + v, ok := d.(atdata.Blob) + if !ok { + return fmt.Errorf("expected a blob") + } + if !(flags&AllowLegacyBlob != 0) && v.Size < 0 { + return fmt.Errorf("legacy blobs not allowed") + } + if len(s.Accept) > 0 { + typeOk := false + for _, pat := range s.Accept { + if acceptableMimeType(pat, v.MimeType) { + typeOk = true + break + } + } + if !typeOk { + return fmt.Errorf("blob mimetype doesn't match accepted: %s", v.MimeType) + } + } + if s.MaxSize != nil && int(v.Size) > *s.MaxSize { + return fmt.Errorf("blob size too large: %d", v.Size) + } + return nil +} + +type SchemaParams struct { + Type string `json:"type"` // "params" + Description *string `json:"description,omitempty"` + Properties map[string]SchemaDef `json:"properties"` // boolean, integer, string, or unknown; or an array of these types + Required []string `json:"required,omitempty"` +} + +func (s *SchemaParams) CheckSchema() error { + if s.Type != "params" { + return fmt.Errorf("expected 'params' schema") + } + // TODO: check for set uniqueness of required + for _, k := range s.Required { + if _, ok := s.Properties[k]; !ok { + return fmt.Errorf("object 'required' field not in properties: %s", k) + } + } + for k, def := range s.Properties { + // TODO: more checks on field name? + if len(k) == 0 { + return fmt.Errorf("empty object schema field name not allowed") + } + switch v := def.Inner.(type) { + case SchemaBoolean, SchemaInteger, SchemaString: + // pass + case SchemaArray: + switch v.Items.Inner.(type) { + case SchemaBoolean, SchemaInteger, SchemaString: + // pass + default: + return fmt.Errorf("params array item type must be boolean, integer, or string") + } + default: + return fmt.Errorf("params field type must be boolean, integer, string, or array") + } + if err := def.CheckSchema(); err != nil { + return err + } + } + return nil +} + +type SchemaToken struct { + Type string `json:"type"` // "token" + Description *string `json:"description,omitempty"` + // the fully-qualified identifier of this token. this is not included in the schema file; it must be added when parsing + FullName string `json:"-"` +} + +func (s *SchemaToken) CheckSchema() error { + if s.Type != "token" { + return fmt.Errorf("expected 'token' schema") + } + if s.FullName == "" { + return fmt.Errorf("expected fully-qualified token name") + } + return nil +} + +func (s *SchemaToken) Validate(d any) error { + return fmt.Errorf("token type does not validate against data") +} + +type SchemaRef struct { + Type string `json:"type"` // "ref" + Description *string `json:"description,omitempty"` + Ref string `json:"ref"` + // full path of reference + fullRef string +} + +func (s *SchemaRef) CheckSchema() error { + if s.Type != "ref" { + return fmt.Errorf("expected 'ref' schema") + } + // TODO: more validation of ref string? + if len(s.Ref) == 0 { + return fmt.Errorf("empty schema ref") + } + if len(s.fullRef) == 0 { + return fmt.Errorf("empty full schema ref") + } + return nil +} + +type SchemaUnion struct { + Type string `json:"type"` // "union" + Description *string `json:"description,omitempty"` + Refs []string `json:"refs"` + Closed *bool `json:"closed,omitempty"` + // fully qualified + fullRefs []string +} + +func (s *SchemaUnion) CheckSchema() error { + if s.Type != "union" { + return fmt.Errorf("expected 'union' schema") + } + if len(s.Refs) == 0 && s.Closed != nil && *s.Closed { + return fmt.Errorf("closed empty unions are not allowed") + } + // TODO: uniqueness check on refs + for _, ref := range s.Refs { + // TODO: more validation of ref string? + if len(ref) == 0 { + return fmt.Errorf("empty schema ref") + } + } + if len(s.fullRefs) != len(s.Refs) { + return fmt.Errorf("union refs were not expanded") + } + return nil +} + +type SchemaUnknown struct { + Type string `json:"type"` // "unknown" + Description *string `json:"description,omitempty"` +} + +func (s *SchemaUnknown) CheckSchema() error { + if s.Type != "unknown" { + return fmt.Errorf("expected 'unknown' schema") + } + return nil +} + +func (s *SchemaUnknown) Validate(d any) error { + _, ok := d.(map[string]any) + if !ok { + return fmt.Errorf("'unknown' data must an object") + } + return nil +} diff --git a/atproto/lexicon/language_test.go b/atproto/lexicon/language_test.go new file mode 100644 index 000000000..cd33396f2 --- /dev/null +++ b/atproto/lexicon/language_test.go @@ -0,0 +1,47 @@ +package lexicon + +import ( + "encoding/json" + "io" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBasicLabelLexicon(t *testing.T) { + assert := assert.New(t) + + f, err := os.Open("testdata/catalog/com_atproto_label_defs.json") + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + + jsonBytes, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + var schema SchemaFile + if err := json.Unmarshal(jsonBytes, &schema); err != nil { + t.Fatal(err) + } + + outBytes, err := json.Marshal(schema) + if err != nil { + t.Fatal(err) + } + + var beforeMap map[string]any + if err := json.Unmarshal(jsonBytes, &beforeMap); err != nil { + t.Fatal(err) + } + + var afterMap map[string]any + if err := json.Unmarshal(outBytes, &afterMap); err != nil { + t.Fatal(err) + } + + assert.Equal(beforeMap, afterMap) +} diff --git a/atproto/lexicon/mimetype.go b/atproto/lexicon/mimetype.go new file mode 100644 index 000000000..5b929962c --- /dev/null +++ b/atproto/lexicon/mimetype.go @@ -0,0 +1,21 @@ +package lexicon + +import ( + "strings" +) + +// checks if val matches pattern, with optional trailing glob on pattern. case-sensitive. +func acceptableMimeType(pattern, val string) bool { + if val == "" || pattern == "" { + return false + } + if pattern == "*/*" { + return true + } + if strings.HasSuffix(pattern, "*") { + prefix := pattern[:len(pattern)-1] + return strings.HasPrefix(val, prefix) + } else { + return pattern == val + } +} diff --git a/atproto/lexicon/mimetype_test.go b/atproto/lexicon/mimetype_test.go new file mode 100644 index 000000000..626ccc673 --- /dev/null +++ b/atproto/lexicon/mimetype_test.go @@ -0,0 +1,22 @@ +package lexicon + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAcceptableMimeType(t *testing.T) { + assert := assert.New(t) + + assert.True(acceptableMimeType("image/*", "image/png")) + assert.True(acceptableMimeType("text/plain", "text/plain")) + assert.True(acceptableMimeType("*/*", "text/plain")) + + assert.False(acceptableMimeType("image/*", "text/plain")) + assert.False(acceptableMimeType("text/plain", "image/png")) + assert.False(acceptableMimeType("text/plain", "")) + assert.False(acceptableMimeType("", "text/plain")) + + // TODO: application/json, application/json+thing +} diff --git a/atproto/lexicon/resolve.go b/atproto/lexicon/resolve.go new file mode 100644 index 000000000..3bd5db07a --- /dev/null +++ b/atproto/lexicon/resolve.go @@ -0,0 +1,87 @@ +package lexicon + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + + "github.com/bluesky-social/indigo/api/agnostic" + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Low-level routine for resolving an NSID to full Lexicon data record (as stored in a repository). +// +// The current implementation uses a naive 'getRepo' fetch to the relevant PDS instance, without validating MST proof chain. +// +// Calling code should usually use ResolvingCatalog, which handles basic caching and validation of the Lexicon language itself. +func ResolveLexiconData(ctx context.Context, dir identity.Directory, nsid syntax.NSID) (map[string]any, error) { + + record, err := resolveLexiconJSON(ctx, dir, nsid) + if err != nil { + return nil, err + } + + d, err := atdata.UnmarshalJSON(*record) + if err != nil { + return nil, fmt.Errorf("fetched Lexicon schema record was invalid: %w", err) + } + return d, nil +} + +// Low-level routine for resolving an NSID to `SchemaFile`. +// +// Same as `ResolveLexiconData`, but returns a parsed `SchemaFile` struct. +func ResolveLexiconSchemaFile(ctx context.Context, dir identity.Directory, nsid syntax.NSID) (*SchemaFile, error) { + record, err := resolveLexiconJSON(ctx, dir, nsid) + if err != nil { + return nil, err + } + + var sf SchemaFile + if err := json.Unmarshal(*record, &sf); err != nil { + return nil, fmt.Errorf("fetched Lexicon schema record was invalid: %w", err) + } + return &sf, nil +} + +// internal helper for fetching lexicon record as JSON bytes +func resolveLexiconJSON(ctx context.Context, dir identity.Directory, nsid syntax.NSID) (*json.RawMessage, error) { + baseDir := identity.BaseDirectory{} + did, err := baseDir.ResolveNSID(ctx, nsid) + if err != nil { + return nil, err + } + slog.Debug("resolved NSID", "nsid", nsid, "did", did) + + ident, err := dir.LookupDID(ctx, did) + if err != nil { + return nil, err + } + + aturi := syntax.ATURI(fmt.Sprintf("at://%s/com.atproto.lexicon.schema/%s", did, nsid)) + msg, err := fetchRecordJSON(ctx, *ident, aturi) + if err != nil { + return nil, err + } + return msg, err +} + +func fetchRecordJSON(ctx context.Context, ident identity.Identity, aturi syntax.ATURI) (*json.RawMessage, error) { + + slog.Debug("fetching record", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String()) + client := atclient.NewAPIClient(ident.PDSEndpoint()) + resp, err := agnostic.RepoGetRecord(ctx, client, "", aturi.Collection().String(), ident.DID.String(), aturi.RecordKey().String()) + if err != nil { + return nil, err + } + + if nil == resp.Value { + return nil, fmt.Errorf("empty record in response") + } + + return resp.Value, nil +} diff --git a/atproto/lexicon/resolving_catalog.go b/atproto/lexicon/resolving_catalog.go new file mode 100644 index 000000000..9ae6d51df --- /dev/null +++ b/atproto/lexicon/resolving_catalog.go @@ -0,0 +1,74 @@ +package lexicon + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Catalog which supplements an in-memory BaseCatalog with live resolution from the network +type ResolvingCatalog struct { + Base BaseCatalog + Directory identity.Directory +} + +func NewResolvingCatalog() ResolvingCatalog { + return ResolvingCatalog{ + Base: NewBaseCatalog(), + Directory: identity.DefaultDirectory(), + } +} + +func (rc *ResolvingCatalog) Resolve(ref string) (*Schema, error) { + // NOTE: not passed through! + ctx := context.Background() + + if ref == "" { + return nil, fmt.Errorf("tried to resolve empty string name") + } + + // first try existing catalog + schema, err := rc.Base.Resolve(ref) + if nil == err { // no error: found a hit + return schema, nil + } + + // split any ref from the end '#' + parts := strings.SplitN(ref, "#", 2) + nsid, err := syntax.ParseNSID(parts[0]) + if err != nil { + return nil, err + } + + record, err := ResolveLexiconData(ctx, rc.Directory, nsid) + if err != nil { + return nil, err + } + + recordJSON, err := json.Marshal(record) + if err != nil { + return nil, err + } + + var sf SchemaFile + if err = json.Unmarshal(recordJSON, &sf); err != nil { + return nil, err + } + + if sf.Lexicon != 1 { + return nil, fmt.Errorf("unsupported lexicon language version: %d", sf.Lexicon) + } + if sf.ID != nsid.String() { + return nil, fmt.Errorf("lexicon ID does not match NSID: %s != %s", sf.ID, nsid) + } + if err = rc.Base.AddSchemaFile(sf); err != nil { + return nil, err + } + + // re-resolving from the raw ref ensures that fragments are handled + return rc.Base.Resolve(ref) +} diff --git a/atproto/lexicon/schemafile.go b/atproto/lexicon/schemafile.go new file mode 100644 index 000000000..6aaa5b972 --- /dev/null +++ b/atproto/lexicon/schemafile.go @@ -0,0 +1,74 @@ +package lexicon + +import ( + "fmt" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Serialization helper type for top-level Lexicon schema JSON objects (files). +// +// Note that the [FinishParse] method should always be called after unmarshalling a SchemaFile from JSON. +type SchemaFile struct { + Type string `json:"$type,omitempty"` // com.atproto.lexicon.schema (if a record) + Lexicon int `json:"lexicon"` // must be 1 + ID string `json:"id"` + Description *string `json:"description,omitempty"` + Defs map[string]SchemaDef `json:"defs"` +} + +// Helper method which should always be called after parsing a schema file (eg, from JSON). +// +// Does some very basic validation (eg, lexicon language version), and fills in +// internal references (for example full name of tokens). +func (sf *SchemaFile) FinishParse() error { + if sf.Lexicon != 1 { + return fmt.Errorf("unsupported lexicon language version: %d", sf.Lexicon) + } + base := sf.ID + for frag, def := range sf.Defs { + if len(frag) == 0 || strings.Contains(frag, "#") || strings.Contains(frag, ".") { + // TODO: more validation here? + return fmt.Errorf("schema name invalid: %s", frag) + } + name := base + "#" + frag + switch s := def.Inner.(type) { + case SchemaToken: + // add fully-qualified name to token + s.FullName = name + def.Inner = s + } + def.setBase(base) + sf.Defs[frag] = def + } + return nil +} + +// Calls [SchemaDef.CheckSchema] recursively over all defs +func (sf *SchemaFile) CheckSchema() error { + if sf.Lexicon != 1 { + return fmt.Errorf("unsupported lexicon language version: %d", sf.Lexicon) + } + + if _, err := syntax.ParseNSID(sf.ID); err != nil { + return fmt.Errorf("invalid lexicon schema NSID: %s", sf.ID) + } + + for frag, def := range sf.Defs { + if len(frag) == 0 || strings.Contains(frag, "#") || strings.Contains(frag, ".") { + // TODO: more validation here? + return fmt.Errorf("schema def name invalid: %s", frag) + } + if def.IsPrimary() && frag != "main" { + return fmt.Errorf("'primary' definition types can only have name 'main', not: %s", frag) + } + if !def.IsDefinable() { + return fmt.Errorf("schema type can not be used as a named definition: %s", frag) + } + if err := def.CheckSchema(); err != nil { + return err + } + } + return nil +} diff --git a/atproto/lexicon/testdata/catalog/com_atproto_label_defs.json b/atproto/lexicon/testdata/catalog/com_atproto_label_defs.json new file mode 100644 index 000000000..f8677dc37 --- /dev/null +++ b/atproto/lexicon/testdata/catalog/com_atproto_label_defs.json @@ -0,0 +1,78 @@ +{ + "lexicon": 1, + "id": "com.atproto.label.defs", + "defs": { + "label": { + "description": "Metadata tag on an atproto resource (eg, repo or record)", + "properties": { + "cid": { + "description": "optionally, CID specifying the specific version of 'uri' resource this label applies to", + "format": "cid", + "type": "string" + }, + "cts": { + "description": "timestamp when this label was created", + "format": "datetime", + "type": "string" + }, + "neg": { + "description": "if true, this is a negation label, overwriting a previous label", + "type": "boolean" + }, + "src": { + "description": "DID of the actor who created this label", + "format": "did", + "type": "string" + }, + "uri": { + "description": "AT URI of the record, repository (account), or other resource which this label applies to", + "format": "uri", + "type": "string" + }, + "val": { + "description": "the short string name of the value or type of this label", + "maxLength": 128, + "type": "string" + } + }, + "required": [ + "src", + "uri", + "val", + "cts" + ], + "type": "object" + }, + "selfLabel": { + "description": "Metadata tag on an atproto record, published by the author within the record. Note -- schemas should use #selfLabels, not #selfLabel.", + "properties": { + "val": { + "description": "the short string name of the value or type of this label", + "maxLength": 128, + "type": "string" + } + }, + "required": [ + "val" + ], + "type": "object" + }, + "selfLabels": { + "description": "Metadata tags on an atproto record, published by the author within the record.", + "properties": { + "values": { + "items": { + "ref": "#selfLabel", + "type": "ref" + }, + "maxLength": 10, + "type": "array" + } + }, + "required": [ + "values" + ], + "type": "object" + } + } +} diff --git a/atproto/lexicon/testdata/catalog/minimal-procedure.json b/atproto/lexicon/testdata/catalog/minimal-procedure.json new file mode 100644 index 000000000..e94d627ab --- /dev/null +++ b/atproto/lexicon/testdata/catalog/minimal-procedure.json @@ -0,0 +1,16 @@ +{ + "lexicon": 1, + "id": "example.lexicon.minimal.procedure", + "description": "demonstrates lexicon features for the procedure type", + "defs": { + "main": { + "type": "procedure", + "input": { + "encoding": "application/json", + "schema": { + "type": "object" + } + } + } + } +} diff --git a/atproto/lexicon/testdata/catalog/minimal-query.json b/atproto/lexicon/testdata/catalog/minimal-query.json new file mode 100644 index 000000000..1f1cc82c3 --- /dev/null +++ b/atproto/lexicon/testdata/catalog/minimal-query.json @@ -0,0 +1,11 @@ +{ + "lexicon": 1, + "id": "example.lexicon.minimal.query", + "description": "exercises many lexicon features for the query type", + "defs": { + "main": { + "type": "query", + "description": "a query type" + } + } +} diff --git a/atproto/lexicon/testdata/catalog/permission-set.json b/atproto/lexicon/testdata/catalog/permission-set.json new file mode 100644 index 000000000..a2079ab9e --- /dev/null +++ b/atproto/lexicon/testdata/catalog/permission-set.json @@ -0,0 +1,79 @@ +{ + "lexicon": 1, + "id": "example.lexicon.permissionset", + "description": "exercises many lexicon features for the permission-set type", + "defs": { + "main": { + "type": "permission-set", + "title": "Example for Moderation", + "title:lang": { + "fr": "Example for Modération" + }, + "detail": "Create moderation reports", + "detail:lang": { + "fr-FR": "Créer des rapports de modération" + }, + "permissions": [ + { + "type": "permission", + "resource": "repo", + "collection": [ + "com.example.calendar.event", + "com.example.calendar.rsvp" + ], + "action": [ + "delete", + "create" + ] + }, + { + "type": "permission", + "resource": "repo", + "collection": [ + "com.example.calendar.event", + "app.bsky.feed.post" + ], + "action": [ + "create", + "update", + "delete" + ] + }, + { + "type": "permission", + "resource": "repo", + "collection": [ + "com.example.calendar.eventV2" + ], + "action": [ + "create" + ] + }, + { + "type": "permission", + "resource": "rpc", + "lxm": [ + "com.example.calendar.listEvents" + ], + "aud": "*" + }, + { + "type": "permission", + "resource": "rpc", + "lxm": [ + "*" + ], + "inheritAud": true + }, + { + "type": "permission", + "resource": "rpc", + "lxm": [ + "com.example.calendar.listEvents" + ], + "inheritAud": true + } + ] + } + } +} diff --git a/atproto/lexicon/testdata/catalog/procedure.json b/atproto/lexicon/testdata/catalog/procedure.json new file mode 100644 index 000000000..4829a3abf --- /dev/null +++ b/atproto/lexicon/testdata/catalog/procedure.json @@ -0,0 +1,78 @@ +{ + "lexicon": 1, + "id": "example.lexicon.procedure", + "defs": { + "main": { + "type": "procedure", + "description": "demonstrates lexicon features for the procedure type", + "parameters": { + "type": "params", + "properties": { + "boolean": { + "type": "boolean", + "description": "field of type boolean" + }, + "integer": { + "type": "integer", + "description": "field of type integer" + }, + "stringField": { + "type": "string", + "description": "field of type string" + } + } + }, + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "preferences" + ], + "properties": { + "preferences": { + "type": "ref", + "ref": "app.bsky.actor.defs#preferences" + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [], + "properties": { + "blob": { + "type": "blob", + "description": "field of type blob" + }, + "unknown": { + "type": "unknown", + "description": "field of type unknown" + }, + "array": { + "type": "array", + "description": "field of type array", + "items": { + "type": "integer" + } + }, + "object": { + "type": "object", + "description": "field of type null", + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + } + } + } + } + } + } + } +} diff --git a/atproto/lexicon/testdata/catalog/query.json b/atproto/lexicon/testdata/catalog/query.json new file mode 100644 index 000000000..c355d0eac --- /dev/null +++ b/atproto/lexicon/testdata/catalog/query.json @@ -0,0 +1,69 @@ +{ + "lexicon": 1, + "id": "example.lexicon.query", + "description": "exercises many lexicon features for the query type", + "defs": { + "main": { + "type": "query", + "description": "a query type", + "parameters": { + "type": "params", + "description": "a params type", + "required": [ + "stringField" + ], + "properties": { + "boolean": { + "type": "boolean", + "description": "field of type boolean" + }, + "integer": { + "type": "integer", + "description": "field of type integer" + }, + "stringField": { + "type": "string", + "description": "field of type string" + }, + "handle": { + "type": "string", + "format": "handle", + "description": "field of type string, format handle" + }, + "array": { + "type": "array", + "description": "field of type array", + "items": { + "type": "integer" + } + } + } + }, + "output": { + "description": "output body type", + "encoding": "application/json", + "schema": { + "type": "object", + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + } + } + }, + "errors": [ + { + "name": "DemoError", + "description": "demo error value" + }, + { + "name": "AnotherDemoError", + "description": "another demo error value" + } + ] + } + } +} diff --git a/atproto/lexicon/testdata/catalog/record.json b/atproto/lexicon/testdata/catalog/record.json new file mode 100644 index 000000000..efe52c0c3 --- /dev/null +++ b/atproto/lexicon/testdata/catalog/record.json @@ -0,0 +1,257 @@ +{ + "lexicon": 1, + "id": "example.lexicon.record", + "description": "demonstrates lexicon features for the record type", + "defs": { + "main": { + "type": "record", + "key": "literal:demo", + "description": "a record type with many field", + "record": { + "type": "object", + "required": [ + "integer" + ], + "nullable": [ + "nullableString" + ], + "properties": { + "boolean": { + "type": "boolean", + "description": "field of type boolean" + }, + "integer": { + "type": "integer", + "description": "field of type integer" + }, + "string": { + "type": "string", + "description": "field of type string" + }, + "nullableString": { + "type": "string", + "description": "field of type string; value is nullable" + }, + "bytes": { + "type": "bytes", + "description": "field of type bytes" + }, + "cid-link": { + "type": "cid-link", + "description": "field of type cid-link" + }, + "blob": { + "type": "blob", + "description": "field of type blob" + }, + "unknown": { + "type": "unknown", + "description": "field of type unknown" + }, + "array": { + "type": "array", + "description": "field of type array", + "items": { + "type": "integer" + } + }, + "object": { + "type": "object", + "description": "field of type object", + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + } + }, + "ref": { + "type": "ref", + "description": "field of type ref", + "ref": "example.lexicon.record#demoObject" + }, + "union": { + "type": "union", + "refs": [ + "example.lexicon.record#demoObject", + "example.lexicon.record#demoObjectTwo" + ] + }, + "formats": { + "type": "ref", + "ref": "example.lexicon.record#stringFormats" + }, + "constInteger": { + "type": "integer", + "const": 42 + }, + "defaultInteger": { + "type": "integer", + "default": 42 + }, + "enumInteger": { + "type": "integer", + "enum": [ + 4, + 9, + 16, + 25 + ] + }, + "rangeInteger": { + "type": "integer", + "minimum": 10, + "maximum": 20 + }, + "lenString": { + "type": "string", + "minLength": 10, + "maxLength": 20 + }, + "graphemeString": { + "type": "string", + "minGraphemes": 10, + "maxGraphemes": 20 + }, + "enumString": { + "type": "string", + "enum": [ + "fish", + "tree", + "rock" + ] + }, + "knownString": { + "type": "string", + "knownValues": [ + "blue", + "green", + "red" + ] + }, + "sizeBytes": { + "type": "bytes", + "minLength": 10, + "maxLength": 20 + }, + "lenArray": { + "type": "array", + "items": { + "type": "integer" + }, + "minLength": 2, + "maxLength": 5 + }, + "sizeBlob": { + "type": "blob", + "maxSize": 20 + }, + "acceptBlob": { + "type": "blob", + "accept": [ + "image/*" + ] + }, + "closedUnion": { + "type": "union", + "refs": [ + "example.lexicon.record#demoObject" + ], + "closed": true + } + } + } + }, + "stringFormats": { + "type": "object", + "description": "all the various string format types", + "properties": { + "did": { + "type": "string", + "format": "did", + "description": "a did string" + }, + "handle": { + "type": "string", + "format": "handle", + "description": "a did string" + }, + "atidentifier": { + "type": "string", + "format": "at-identifier", + "description": "an at-identifier string" + }, + "nsid": { + "type": "string", + "format": "nsid", + "description": "an nsid string" + }, + "aturi": { + "type": "string", + "format": "at-uri", + "description": "an at-uri string" + }, + "cid": { + "type": "string", + "format": "cid", + "description": "a cid string (not a cid-link)" + }, + "datetime": { + "type": "string", + "format": "datetime", + "description": "a datetime string" + }, + "language": { + "type": "string", + "format": "language", + "description": "a language string" + }, + "uri": { + "type": "string", + "format": "uri", + "description": "a generic URI field" + }, + "tid": { + "type": "string", + "format": "tid", + "description": "a generic TID field" + }, + "recordkey": { + "type": "string", + "format": "record-key", + "description": "a generic record-key field" + } + } + }, + "demoToken": { + "type": "token", + "description": "an example of what a token looks like" + }, + "demoObject": { + "type": "object", + "description": "smaller object schema for unions", + "properties": { + "a": { + "type": "integer" + }, + "b": { + "type": "integer" + } + } + }, + "demoObjectTwo": { + "type": "object", + "description": "smaller object schema for unions", + "properties": { + "c": { + "type": "integer" + }, + "d": { + "type": "integer" + } + } + } + } +} diff --git a/atproto/lexicon/testdata/catalog/subscription.json b/atproto/lexicon/testdata/catalog/subscription.json new file mode 100644 index 000000000..96da2fe10 --- /dev/null +++ b/atproto/lexicon/testdata/catalog/subscription.json @@ -0,0 +1,48 @@ +{ + "lexicon": 1, + "id": "example.lexicon.subscription", + "description": "demonstrates lexicon features for the subscription type", + "defs": { + "main": { + "type": "subscription", + "description": "an example event stream", + "parameters": { + "type": "params", + "properties": { + "cursor": { + "type": "integer", + "description": "start at the given sequence number" + } + } + }, + "message": { + "schema": { + "type": "union", + "refs": ["#yo", "#info"] + } + }, + "errors": [{ "name": "FutureCursor" }] + }, + "yo": { + "type": "object", + "required": ["seq", "yo"], + "properties": { + "seq": { "type": "integer" }, + "yo": { "type": "boolean" } + } + }, + "info": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string", + "knownValues": ["OutdatedCursor"] + }, + "message": { + "type": "string" + } + } + } + } +} diff --git a/atproto/lexicon/testdata/lexicon-invalid.json b/atproto/lexicon/testdata/lexicon-invalid.json new file mode 100644 index 000000000..cb612bee8 --- /dev/null +++ b/atproto/lexicon/testdata/lexicon-invalid.json @@ -0,0 +1,99 @@ +[ + { + "name": "invalid lexicon field", + "lexicon": { + "lexicon": "one", + "id": "example.lexicon.other", + "defs": { + "demo": { + "type": "integer" + } + } + } + }, + { + "name": "invalid id field", + "lexicon": { + "lexicon": 1, + "id": 2, + "defs": { + "demo": { + "type": "integer" + } + } + } + }, + { + "name": "invalid NSID", + "lexicon": { + "lexicon": 1, + "id": "one-two-three", + "defs": { + "demo": { + "type": "integer" + } + } + } + }, + { + "name": "defined unknown", + "lexicon": { + "lexicon": 1, + "id": "example.lexicon.other", + "defs": { + "demo": { + "type": "unknown" + } + } + } + }, + { + "name": "defined ref", + "lexicon": { + "lexicon": 1, + "id": "example.lexicon.other", + "defs": { + "demo": { + "type": "ref", + "ref": "com.atproto.repo.strongRef" + } + } + } + }, + { + "name": "non-main primary", + "lexicon": { + "lexicon": 1, + "id": "example.lexicon.other", + "defs": { + "demo": { + "type": "record", + "key": "any", + "record": { + "type": "object" + } + } + } + } + }, + { + "name": "record missing type object", + "lexicon": { + "lexicon": 1, + "id": "example.lexicon.other", + "defs": { + "main": { + "type": "record", + "key": "any", + "record": { + "properties": { + "b": { + "type": "boolean" + } + } + } + } + } + } + } +] diff --git a/atproto/lexicon/testdata/lexicon-valid.json b/atproto/lexicon/testdata/lexicon-valid.json new file mode 100644 index 000000000..fd0c03ae8 --- /dev/null +++ b/atproto/lexicon/testdata/lexicon-valid.json @@ -0,0 +1,76 @@ +[ + { + "name": "minimal", + "lexicon": { + "lexicon": 1, + "id": "example.lexicon.other", + "defs": { + "demo": { + "type": "integer" + } + } + } + }, + { + "name": "minimal record", + "lexicon": { + "lexicon": 1, + "id": "example.lexicon.record", + "defs": { + "main": { + "type": "record", + "key": "any", + "record": { + "type": "object", + "properties": {} + } + } + } + } + }, + { + "name": "basic permission-set", + "lexicon": { + "lexicon": 1, + "id": "example.lexicon.perms", + "defs": { + "main": { + "type": "permission-set", + "title": "test case", + "permissions": [ + { + "type": "permission", + "resource": "repo", + "collection": [ + "com.example.calendar.event" + ], + "action": [ + "delete", + "create" + ] + }, + { + "type": "permission", + "resource": "repo", + "collection": [ + "com.example.calendar.rsvp" + ] + }, + { + "type": "permission", + "resource": "rpc", + "lxm": ["example.lexicon.endpoint"], + "aud": "*" + }, + { + "type": "permission", + "resource": "rpc", + "lxm": ["example.lexicon.endpointTwo"], + "inheritAud": true + } + ] + } + } + } + } +] diff --git a/atproto/lexicon/testdata/record-data-invalid.json b/atproto/lexicon/testdata/record-data-invalid.json new file mode 100644 index 000000000..17af3e7dc --- /dev/null +++ b/atproto/lexicon/testdata/record-data-invalid.json @@ -0,0 +1,534 @@ +[ + { + "name": "missing required field", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record" + } + }, + { + "name": "invalid boolean field", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "boolean": "green" + } + }, + { + "name": "invalid integer field", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": "green" + } + }, + { + "name": "invalid non-nullable string field", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "string": null + } + }, + { + "name": "invalid string field", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "string": 2 + } + }, + { + "name": "invalid bytes field", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "bytes": "green" + } + }, + { + "name": "invalid bytes: empty object", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "bytes": {} + } + }, + { + "name": "invalid bytes: wrong type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "bytes": { + "bytes": "asdfasdfasdfasdf" + } + } + }, + { + "name": "invalid cid-link field", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "cid-link": "green" + } + }, + { + "name": "invalid blob field", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "blob": "green" + } + }, + { + "name": "invalid blob: wrong type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "blob": { + "type": "blob", + "size": 123, + "mimeType": false, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + } + } + }, + { + "name": "invalid array", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "array": 123 + } + }, + { + "name": "invalid array element", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "array": [ + true, + false + ] + } + }, + { + "name": "object wrong data type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "object": 123 + } + }, + { + "name": "object nested wrong data type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "object": { + "a": "not-a-number" + } + } + }, + { + "name": "invalid token ref type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "ref": 123 + } + }, + { + "name": "invalid ref value", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "ref": "example.lexicon.record#wrongToken" + } + }, + { + "name": "invalid string format handle", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "handle": "123" + } + } + }, + { + "name": "invalid string format did", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "did": "123" + } + } + }, + { + "name": "invalid string format atidentifier", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "atidentifier": "123" + } + } + }, + { + "name": "invalid string format nsid", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "nsid": "123" + } + } + }, + { + "name": "invalid string format aturi", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "aturi": "123" + } + } + }, + { + "name": "invalid string format cid", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "cid": "123" + } + } + }, + { + "name": "invalid string format datetime", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "datetime": "123" + } + } + }, + { + "name": "invalid string format language", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "language": "123" + } + } + }, + { + "name": "invalid string format uri", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "uri": "123" + } + } + }, + { + "name": "invalid string format tid", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "tid": "000" + } + } + }, + { + "name": "invalid string format recordkey", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "formats": { + "recordkey": "." + } + } + }, + { + "name": "wrong const value", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "constInteger": 41 + } + }, + { + "name": "integer not in enum", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "enumInteger": 7 + } + }, + { + "name": "out of integer range", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "rangeInteger": 9000 + } + }, + { + "name": "string too short", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "lenString": "." + } + }, + { + "name": "string too long", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "lenString": "abcdefg-abcdefg-abcdefg" + } + }, + { + "name": "string too short (graphemes)", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "graphemeString": "👩‍👩‍👦‍👦👩‍👩‍👦‍👦" + } + }, + { + "name": "string too long (graphemes)", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "graphemeString": "abcdefg-abcdefg-abcdefg" + } + }, + { + "name": "out of enum string", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "enumString": "unexpected" + } + }, + { + "name": "bytes too short", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "sizeBytes": { + "$bytes": "b25l" + } + } + }, + { + "name": "bytes too long", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "sizeBytes": { + "$bytes": "b25lb25lb25lb25lb25lb25lb25lb25lb25lb25lb25l" + } + } + }, + { + "name": "array too short", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "lenArray": [ + 0 + ] + } + }, + { + "name": "array too long", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "lenArray": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + } + }, + { + "name": "blob too large", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "sizeBlob": { + "$type": "blob", + "size": 12345, + "mimeType": "text/plain", + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + } + } + }, + { + "name": "blob wrong type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "acceptBlob": { + "$type": "blob", + "size": 12345, + "mimeType": "text/plain", + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + } + } + }, + { + "name": "open union wrong data type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "union": 123 + } + }, + { + "name": "open union missing $type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "union": { + "a": 1, + "b": 2 + } + } + }, + { + "name": "out of closed union", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "closedUnion": { + "$type": "example.unknown-lexicon.blah", + "a": 1 + } + } + }, + { + "name": "union inner invalid", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "closedUnion": { + "$type": "example.lexicon.record#demoObjectTwo", + "a": 1 + } + } + }, + { + "name": "union inner invalid", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "union": { + "$type": "example.lexicon.record#demoObject", + "a": "not-a-number" + } + } + }, + { + "name": "unknown wrong type (bool)", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "unknown": false + } + }, + { + "name": "unknown wrong type (bytes)", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "unknown": { + "$bytes": "123" + } + } + }, + { + "name": "unknown wrong type (blob)", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "unknown": { + "$type": "blob", + "mimeType": "text/plain", + "size": 12345, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + } + } + } +] diff --git a/atproto/lexicon/testdata/record-data-valid.json b/atproto/lexicon/testdata/record-data-valid.json new file mode 100644 index 000000000..3eb0e0232 --- /dev/null +++ b/atproto/lexicon/testdata/record-data-valid.json @@ -0,0 +1,118 @@ +[ + { + "name": "minimal", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1 + } + }, + { + "name": "full", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "boolean": true, + "integer": 3, + "string": "blah", + "nullableString": null, + "bytes": { + "$bytes": "123" + }, + "cidlink": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + }, + "blob": { + "$type": "blob", + "mimeType": "text/plain", + "size": 12345, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }, + "unknown": { + "a": "alphabet", + "b": 3 + }, + "array": [ + 1, + 2, + 3 + ], + "object": { + "a": 1, + "b": 2 + }, + "ref": { + "a": 1, + "b": 2 + }, + "union": { + "$type": "example.lexicon.record#demoObject", + "a": 1, + "b": 2 + }, + "formats": { + "did": "did:web:example.com", + "handle": "handle.example.com", + "atidentifier": "handle.example.com", + "aturi": "at://handle.example.com/com.example.nsid/asdf123", + "nsid": "com.example.nsid", + "cid": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq", + "datetime": "2023-10-30T22:25:23Z", + "language": "en", + "tid": "3kznmn7xqxl22", + "recordkey": "simple" + }, + "constInteger": 42, + "defaultInteger": 123, + "enumInteger": 16, + "rangeInteger": 16, + "lenString": "1234567890ABC", + "graphemeString": "🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈🇩🇪🏳️‍🌈", + "enumString": "fish", + "knownString": "blue", + "sizeBytes": { + "$bytes": "asdfasdfasdfasdf" + }, + "lenArray": [ + 1, + 2, + 3 + ], + "sizeBlob": { + "$type": "blob", + "mimeType": "text/plain", + "size": 8, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }, + "acceptBlob": { + "$type": "blob", + "mimeType": "image/png", + "size": 12345, + "ref": { + "$link": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq" + } + }, + "closedUnion": { + "$type": "example.lexicon.record#demoObject", + "a": 1 + } + } + }, + { + "name": "unknown as a type", + "rkey": "demo", + "data": { + "$type": "example.lexicon.record", + "integer": 1, + "unknown": { + "$type": "example.lexicon.record#demoObject", + "a": 1, + "b": 2 + } + } + } +] diff --git a/atproto/lexicon/validation.go b/atproto/lexicon/validation.go new file mode 100644 index 000000000..c227b06d3 --- /dev/null +++ b/atproto/lexicon/validation.go @@ -0,0 +1,176 @@ +package lexicon + +import ( + "fmt" + "reflect" +) + +// Boolean flags tweaking how Lexicon validation rules are interpreted. +type ValidateFlags int + +const ( + // Flag which allows legacy "blob" data to pass validation. + AllowLegacyBlob = 1 << iota + // Flag which loosens "datetime" string syntax validation. String must still be an ISO datetime, but might be missing timezone (for example) + AllowLenientDatetime + // Flag which requires validation of nested data in open unions. By default nested union types are only validated optimistically (if the type is known in catatalog) for unlisted types. This flag will result in a validation error if the Lexicon can't be resolved from the catalog. + StrictRecursiveValidation +) + +// Combination of argument flags for less formal validation. Recommended for, eg, working with old/legacy data from 2023. +var LenientMode ValidateFlags = AllowLegacyBlob | AllowLenientDatetime + +// Represents a Lexicon schema definition +type Schema struct { + ID string + Def any +} + +// Checks Lexicon schema (fetched from the catalog) for the given record, with optional flags tweaking default validation rules. +// +// 'recordData' is typed as 'any', but is expected to be 'map[string]any' +// 'ref' is a reference to the schema type, as an NSID with optional fragment. For records, the '$type' must match 'ref' +// 'flags' are parameters tweaking Lexicon validation rules. Zero value is default. +func ValidateRecord(cat Catalog, recordData any, ref string, flags ValidateFlags) error { + return validateRecordConfig(cat, recordData, ref, flags) +} + +func validateRecordConfig(cat Catalog, recordData any, ref string, flags ValidateFlags) error { + def, err := cat.Resolve(ref) + if err != nil { + return err + } + s, ok := def.Def.(SchemaRecord) + if !ok { + return fmt.Errorf("schema is not of record type: %s", ref) + } + d, ok := recordData.(map[string]any) + if !ok { + return fmt.Errorf("record data is not object type") + } + t, ok := d["$type"] + if !ok || t != ref { + return fmt.Errorf("record data missing $type, or didn't match expected NSID") + } + return validateObject(cat, s.Record, d, flags) +} + +func validateData(cat Catalog, def any, d any, flags ValidateFlags) error { + switch v := def.(type) { + case SchemaBoolean: + return v.Validate(d) + case SchemaInteger: + return v.Validate(d) + case SchemaString: + return v.Validate(d, flags) + case SchemaBytes: + return v.Validate(d) + case SchemaCIDLink: + return v.Validate(d) + case SchemaArray: + arr, ok := d.([]any) + if !ok { + return fmt.Errorf("expected an array, got: %s", reflect.TypeOf(d)) + } + return validateArray(cat, v, arr, flags) + case SchemaObject: + obj, ok := d.(map[string]any) + if !ok { + return fmt.Errorf("expected an object, got: %s", reflect.TypeOf(d)) + } + return validateObject(cat, v, obj, flags) + case SchemaBlob: + return v.Validate(d, flags) + case SchemaRef: + // recurse + next, err := cat.Resolve(v.fullRef) + if err != nil { + return err + } + return validateData(cat, next.Def, d, flags) + case SchemaUnion: + return validateUnion(cat, v, d, flags) + case SchemaUnknown: + return v.Validate(d) + case SchemaToken: + return v.Validate(d) + default: + return fmt.Errorf("unhandled schema type: %s", reflect.TypeOf(v)) + } +} + +func validateObject(cat Catalog, s SchemaObject, d map[string]any, flags ValidateFlags) error { + for _, k := range s.Required { + if _, ok := d[k]; !ok { + return fmt.Errorf("required field missing: %s", k) + } + } + for k, def := range s.Properties { + if v, ok := d[k]; ok { + if v == nil && s.IsNullable(k) { + continue + } + err := validateData(cat, def.Inner, v, flags) + if err != nil { + return err + } + } + } + return nil +} + +func validateArray(cat Catalog, s SchemaArray, arr []any, flags ValidateFlags) error { + if (s.MinLength != nil && len(arr) < *s.MinLength) || (s.MaxLength != nil && len(arr) > *s.MaxLength) { + return fmt.Errorf("array length out of bounds: %d", len(arr)) + } + for _, v := range arr { + err := validateData(cat, s.Items.Inner, v, flags) + if err != nil { + return err + } + } + return nil +} + +func validateUnion(cat Catalog, s SchemaUnion, d any, flags ValidateFlags) error { + closed := s.Closed != nil && *s.Closed == true + + obj, ok := d.(map[string]any) + if !ok { + return fmt.Errorf("union data is not object type") + } + typeVal, ok := obj["$type"] + if !ok { + return fmt.Errorf("union data must have $type") + } + t, ok := typeVal.(string) + if !ok { + return fmt.Errorf("union data must have string $type") + } + + for _, ref := range s.fullRefs { + if ref != t { + continue + } + def, err := cat.Resolve(ref) + if err != nil { + return fmt.Errorf("could not resolve known union variant $type: %s", ref) + } + return validateData(cat, def.Def, d, flags) + } + if closed { + return fmt.Errorf("data did not match any variant of closed union: %s", t) + } + + // eagerly attempt validation of the open union type + // TODO: validate reference as NSID with optional fragment + def, err := cat.Resolve(t) + if err != nil { + if flags&StrictRecursiveValidation != 0 { + return fmt.Errorf("could not strictly validate open union variant $type: %s", t) + } + // by default, ignore validation of unknown open union data + return nil + } + return validateData(cat, def.Def, d, flags) +} diff --git a/atproto/lexicon/validation_test.go b/atproto/lexicon/validation_test.go new file mode 100644 index 000000000..ce5c490ec --- /dev/null +++ b/atproto/lexicon/validation_test.go @@ -0,0 +1,47 @@ +package lexicon + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBasicCatalog(t *testing.T) { + assert := assert.New(t) + + cat := NewBaseCatalog() + if err := cat.LoadDirectory("testdata/catalog"); err != nil { + t.Fatal(err) + } + + def, err := cat.Resolve("com.atproto.label.defs#label") + if err != nil { + t.Fatal(err) + } + assert.NoError(validateData( + &cat, + def.Def, + map[string]any{ + "cid": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + "cts": "2000-01-01T00:00:00.000Z", + "neg": false, + "src": "did:example:labeler", + "uri": "at://did:plc:asdf123/com.atproto.feed.post/asdf123", + "val": "test-label", + }, + 0, + )) + + assert.Error(validateData( + &cat, + def.Def, + map[string]any{ + "cid": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + "cts": "2000-01-01T00:00:00.000Z", + "neg": false, + "uri": "at://did:plc:asdf123/com.atproto.feed.post/asdf123", + "val": "test-label", + }, + 0, + )) +} diff --git a/atproto/repo/car.go b/atproto/repo/car.go new file mode 100644 index 000000000..b67f1c83c --- /dev/null +++ b/atproto/repo/car.go @@ -0,0 +1,120 @@ +package repo + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + + "github.com/bluesky-social/indigo/atproto/repo/mst" + "github.com/bluesky-social/indigo/atproto/syntax" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car" +) + +var ErrNoRoot = errors.New("CAR file missing root CID") +var ErrNoCommit = errors.New("no commit") + +func LoadRepoFromCAR(ctx context.Context, r io.Reader) (*Commit, *Repo, error) { + + //bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) + bs := NewTinyBlockstore() + + cr, err := car.NewCarReader(r) + if err != nil { + return nil, nil, err + } + + if cr.Header.Version != 1 { + return nil, nil, fmt.Errorf("unsupported CAR file version: %d", cr.Header.Version) + } + if len(cr.Header.Roots) < 1 { + return nil, nil, fmt.Errorf("CAR file missing root CID") + } + commitCID := cr.Header.Roots[0] + + for { + blk, err := cr.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, nil, err + } + + if err := bs.Put(ctx, blk); err != nil { + return nil, nil, err + } + } + + commitBlock, err := bs.Get(ctx, commitCID) + if err != nil { + return nil, nil, fmt.Errorf("reading commit block from CAR file: %w", err) + } + + var commit Commit + if err := commit.UnmarshalCBOR(bytes.NewReader(commitBlock.RawData())); err != nil { + return nil, nil, fmt.Errorf("parsing commit block from CAR file: %w", err) + } + if err := commit.VerifyStructure(); err != nil { + return nil, nil, fmt.Errorf("parsing commit block from CAR file: %w", err) + } + + tree, err := mst.LoadTreeFromStore(ctx, bs, commit.Data) + if err != nil { + return nil, nil, fmt.Errorf("reading MST from CAR file: %w", err) + } + clk := syntax.ClockFromTID(syntax.TID(commit.Rev)) + repo := Repo{ + DID: syntax.DID(commit.DID), // NOTE: VerifyStructure() already checked DID syntax + Clock: &clk, + MST: *tree, + RecordStore: bs, // TODO: put just records in a smaller blockstore? + } + return &commit, &repo, nil +} + +// LoadCommitFromCAR is like LoadRepoFromCAR() but filters to only return the commit object. +// Also returns the commit CID. +func LoadCommitFromCAR(ctx context.Context, r io.Reader) (*Commit, *cid.Cid, error) { + cr, err := car.NewCarReader(r) + if err != nil { + return nil, nil, err + } + if cr.Header.Version != 1 { + return nil, nil, fmt.Errorf("unsupported CAR file version: %d", cr.Header.Version) + } + if len(cr.Header.Roots) < 1 { + return nil, nil, ErrNoRoot + } + commitCID := cr.Header.Roots[0] + var commitBlock blocks.Block + for { + blk, err := cr.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, nil, err + } + + if blk.Cid().Equals(commitCID) { + commitBlock = blk + break + } + } + if commitBlock == nil { + return nil, nil, ErrNoCommit + } + var commit Commit + if err := commit.UnmarshalCBOR(bytes.NewReader(commitBlock.RawData())); err != nil { + return nil, nil, fmt.Errorf("parsing commit block from CAR file: %w", err) + } + if err := commit.VerifyStructure(); err != nil { + return nil, nil, fmt.Errorf("parsing commit block from CAR file: %w", err) + } + return &commit, &commitCID, nil +} diff --git a/atproto/repo/cbor_gen.go b/atproto/repo/cbor_gen.go new file mode 100644 index 000000000..f46bff4c8 --- /dev/null +++ b/atproto/repo/cbor_gen.go @@ -0,0 +1,340 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package repo + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *Commit) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 6 + + if t.Sig == nil { + fieldCount-- + } + + if t.Rev == "" { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + return err + } + + // t.DID (string) (string) + if len("did") > 1000000 { + return xerrors.Errorf("Value in field \"did\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil { + return err + } + if _, err := cw.WriteString(string("did")); err != nil { + return err + } + + if len(t.DID) > 1000000 { + return xerrors.Errorf("Value in field t.DID was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.DID))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.DID)); err != nil { + return err + } + + // t.Rev (string) (string) + if t.Rev != "" { + + if len("rev") > 1000000 { + return xerrors.Errorf("Value in field \"rev\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("rev"))); err != nil { + return err + } + if _, err := cw.WriteString(string("rev")); err != nil { + return err + } + + if len(t.Rev) > 1000000 { + return xerrors.Errorf("Value in field t.Rev was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Rev))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Rev)); err != nil { + return err + } + } + + // t.Sig ([]uint8) (slice) + if t.Sig != nil { + + if len("sig") > 1000000 { + return xerrors.Errorf("Value in field \"sig\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sig"))); err != nil { + return err + } + if _, err := cw.WriteString(string("sig")); err != nil { + return err + } + + if len(t.Sig) > 2097152 { + return xerrors.Errorf("Byte array in field t.Sig was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Sig))); err != nil { + return err + } + + if _, err := cw.Write(t.Sig); err != nil { + return err + } + + } + + // t.Data (cid.Cid) (struct) + if len("data") > 1000000 { + return xerrors.Errorf("Value in field \"data\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("data"))); err != nil { + return err + } + if _, err := cw.WriteString(string("data")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Data); err != nil { + return xerrors.Errorf("failed to write cid field t.Data: %w", err) + } + + // t.Prev (cid.Cid) (struct) + if len("prev") > 1000000 { + return xerrors.Errorf("Value in field \"prev\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("prev"))); err != nil { + return err + } + if _, err := cw.WriteString(string("prev")); err != nil { + return err + } + + if t.Prev == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Prev); err != nil { + return xerrors.Errorf("failed to write cid field t.Prev: %w", err) + } + } + + // t.Version (int64) (int64) + if len("version") > 1000000 { + return xerrors.Errorf("Value in field \"version\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("version"))); err != nil { + return err + } + if _, err := cw.WriteString(string("version")); err != nil { + return err + } + + if t.Version >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Version)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Version-1)); err != nil { + return err + } + } + + return nil +} + +func (t *Commit) UnmarshalCBOR(r io.Reader) (err error) { + *t = Commit{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Commit: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 7) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.DID (string) (string) + case "did": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.DID = string(sval) + } + // t.Rev (string) (string) + case "rev": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Rev = string(sval) + } + // t.Sig ([]uint8) (slice) + case "sig": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.Sig: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Sig = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Sig); err != nil { + return err + } + + // t.Data (cid.Cid) (struct) + case "data": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Data: %w", err) + } + + t.Data = c + + } + // t.Prev (cid.Cid) (struct) + case "prev": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Prev: %w", err) + } + + t.Prev = &c + } + + } + // t.Version (int64) (int64) + case "version": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Version = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} diff --git a/atproto/repo/cmd/repo-tool/firehose.go b/atproto/repo/cmd/repo-tool/firehose.go new file mode 100644 index 000000000..5120f5bef --- /dev/null +++ b/atproto/repo/cmd/repo-tool/firehose.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/url" + "os" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/repo" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/schedulers/parallel" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/gorilla/websocket" + "github.com/urfave/cli/v3" +) + +// write out error cases as JSON files to disk, for use in regression tests +var CAPTURE_TEST_CASES = false + +func runVerifyFirehose(ctx context.Context, cmd *cli.Command) error { + + slog.SetDefault(configLogger(ctx, cmd, os.Stdout)) + + relayHost := cmd.String("relay-host") + + dialer := websocket.DefaultDialer + u, err := url.Parse(relayHost) + if err != nil { + return fmt.Errorf("invalid relayHost URI: %w", err) + } + u.Path = "xrpc/com.atproto.sync.subscribeRepos" + con, _, err := dialer.Dial(u.String(), http.Header{ + "User-Agent": []string{fmt.Sprintf("at-repo-tool/%s", versioninfo.Short())}, + }) + if err != nil { + return fmt.Errorf("subscribing to firehose failed (dialing): %w", err) + } + + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + slog.Debug("commit event", "did", evt.Repo, "seq", evt.Seq) + return handleCommitEvent(ctx, evt) + }, + } + + scheduler := parallel.NewScheduler( + 1, + 100, + relayHost, + rsc.EventHandler, + ) + slog.Info("starting firehose consumer", "relayHost", relayHost) + return events.HandleRepoStream(ctx, con, scheduler, nil) +} + +func handleCommitEvent(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) error { + // TODO: just log errors, not fail? + _, err := repo.VerifyCommitMessage(ctx, evt) + if err != nil && CAPTURE_TEST_CASES { + body, err := json.MarshalIndent(evt, "", " ") + if err != nil { + return err + } + p := fmt.Sprintf("firehose_commit_%d.json", evt.Seq) + if err := os.WriteFile(p, body, 0600); err != nil { + return err + } + } + return err +} diff --git a/atproto/repo/cmd/repo-tool/main.go b/atproto/repo/cmd/repo-tool/main.go new file mode 100644 index 000000000..44611e475 --- /dev/null +++ b/atproto/repo/cmd/repo-tool/main.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "strings" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/repo" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "repo-tool", + Usage: "development tool for atproto MST trees, CAR files, etc", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "log-level", + Usage: "log verbosity level (eg: warn, info, debug)", + Sources: cli.EnvVars("BEEMO_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"), + }, + }, + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "verify-car-mst", + Usage: "load a CAR file and check the MST tree", + ArgsUsage: "", + Action: runVerifyCarMst, + }, + &cli.Command{ + Name: "verify-car-signature", + Usage: "load a CAR file and check the commit message signature", + ArgsUsage: "", + Action: runVerifyCarSignature, + }, + &cli.Command{ + Name: "verify-firehose", + Usage: "subscribes to sync firehose and validates commit messages", + Action: runVerifyFirehose, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "relay-host", + Usage: "method, hostname, and port of Relay instance (websocket)", + Value: "wss://bsky.network", + Sources: cli.EnvVars("ATP_RELAY_HOST"), + }, + }, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + if err := app.Run(context.Background(), os.Args); err != nil { + slog.Error("command failed", "error", err) + os.Exit(-1) + } +} + +func configLogger(ctx context.Context, cmd *cli.Command, writer io.Writer) *slog.Logger { + var level slog.Level + switch strings.ToLower(cmd.String("log-level")) { + case "error": + level = slog.LevelError + case "warn": + level = slog.LevelWarn + case "info": + level = slog.LevelInfo + case "debug": + level = slog.LevelDebug + default: + level = slog.LevelInfo + } + logger := slog.New(slog.NewJSONHandler(writer, &slog.HandlerOptions{ + Level: level, + })) + slog.SetDefault(logger) + return logger +} + +func runVerifyCarMst(ctx context.Context, cmd *cli.Command) error { + p := cmd.Args().First() + if p == "" { + return fmt.Errorf("need to provide path to CAR file") + } + + f, err := os.Open(p) + if err != nil { + return err + } + defer f.Close() + + commit, repo, err := repo.LoadRepoFromCAR(ctx, f) + if err != nil { + return err + } + + computedCID, err := repo.MST.RootCID() + if err != nil { + return err + } + + if commit.Data != *computedCID { + return fmt.Errorf("failed to re-compute: %s != %s", computedCID, commit.Data) + } + fmt.Println("verified tree") + return nil +} + +func runVerifyCarSignature(ctx context.Context, cmd *cli.Command) error { + dir := identity.DefaultDirectory() + + p := cmd.Args().First() + if p == "" { + return fmt.Errorf("need to provide path to CAR file") + } + + f, err := os.Open(p) + if err != nil { + return err + } + defer f.Close() + + commit, _, err := repo.LoadRepoFromCAR(ctx, f) + if err != nil { + return err + } + + if err := commit.VerifyStructure(); err != nil { + return err + } + did, err := syntax.ParseDID(commit.DID) + if err != nil { + return err + } + + ident, err := dir.LookupDID(ctx, did) + if err != nil { + return err + } + pubkey, err := ident.PublicKey() + if err != nil { + return err + } + if err := commit.VerifySignature(pubkey); err != nil { + return err + } + fmt.Println("verified signature") + return nil +} diff --git a/atproto/repo/commit.go b/atproto/repo/commit.go new file mode 100644 index 000000000..aa364f11d --- /dev/null +++ b/atproto/repo/commit.go @@ -0,0 +1,106 @@ +package repo + +import ( + "bytes" + "fmt" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/ipfs/go-cid" +) + +// atproto repo commit object as a struct type. Can be used for direct CBOR or JSON serialization. +type Commit struct { + DID string `json:"did" cborgen:"did"` + Version int64 `json:"version" cborgen:"version"` // currently: 3 + Prev *cid.Cid `json:"prev" cborgen:"prev"` // NOTE: omitempty would break signature verification for repo v3 + Data cid.Cid `json:"data" cborgen:"data"` + Sig []byte `json:"sig,omitempty" cborgen:"sig,omitempty"` + Rev string `json:"rev,omitempty" cborgen:"rev,omitempty"` +} + +// does basic checks that field values and syntax are correct +func (c *Commit) VerifyStructure() error { + if c.Version != ATPROTO_REPO_VERSION { + return fmt.Errorf("unsupported repo version: %d", c.Version) + } + if len(c.Sig) == 0 { + return fmt.Errorf("empty commit signature") + } + _, err := syntax.ParseDID(c.DID) + if err != nil { + return fmt.Errorf("invalid commit data: %w", err) + } + _, err = syntax.ParseTID(c.Rev) + if err != nil { + return fmt.Errorf("invalid commit data: %w", err) + } + return nil +} + +// returns a representation of the commit object as atproto data (eg, for JSON serialization) +func (c *Commit) AsData() map[string]any { + d := map[string]any{ + "did": c.DID, + "version": c.Version, + "prev": (*atdata.CIDLink)(c.Prev), + "data": atdata.CIDLink(c.Data), + } + if c.Sig != nil { + d["sig"] = atdata.Bytes(c.Sig) + } + if c.Rev != "" { + d["rev"] = c.Rev + } + return d +} + +// Encodes the commit object as DAG-CBOR, without the signature field. Used for signing or validating signatures. +func (c *Commit) UnsignedBytes() ([]byte, error) { + buf := new(bytes.Buffer) + if c.Sig == nil { + if err := c.MarshalCBOR(buf); err != nil { + return nil, err + } + return buf.Bytes(), nil + } + unsigned := Commit{ + DID: c.DID, + Version: c.Version, + Prev: c.Prev, + Data: c.Data, + Rev: c.Rev, + } + if err := unsigned.MarshalCBOR(buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Signs the commit, storing the signature in the `Sig` field +func (c *Commit) Sign(privkey atcrypto.PrivateKey) error { + b, err := c.UnsignedBytes() + if err != nil { + return err + } + sig, err := privkey.HashAndSign(b) + if err != nil { + return err + } + c.Sig = sig + return nil +} + +// Verifies `Sig` field using the provided key. Returns `nil` if signature is valid. +func (c *Commit) VerifySignature(pubkey atcrypto.PublicKey) error { + if c.Sig == nil { + return fmt.Errorf("can not verify unsigned commit") + } + b, err := c.UnsignedBytes() + if err != nil { + return err + } + return pubkey.HashAndVerify(b, c.Sig) +} diff --git a/atproto/repo/doc.go b/atproto/repo/doc.go new file mode 100644 index 000000000..da65ba7d0 --- /dev/null +++ b/atproto/repo/doc.go @@ -0,0 +1,6 @@ +/* +Implementation of atproto repository and sync APIs, built on the MST data structure. + +The current package works for processing a sync firehose, including validation of "inductive firehose". It does not yet work for implementing a repository host (PDS). +*/ +package repo diff --git a/atproto/repo/inductive_interop_test.go b/atproto/repo/inductive_interop_test.go new file mode 100644 index 000000000..27878601b --- /dev/null +++ b/atproto/repo/inductive_interop_test.go @@ -0,0 +1,155 @@ +package repo + +import ( + "context" + "encoding/json" + "os" + "testing" + + "github.com/bluesky-social/indigo/atproto/repo/mst" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/stretchr/testify/assert" +) + +type CommitProofFixture struct { + Comment string `json:"comment"` + LeafValue string `json:"leafValue"` + Keys []string `json:"keys"` + Additions []string `json:"adds"` + Deletions []string `json:"dels"` + RootBeforeCommit string `json:"rootBeforeCommit"` + RootAfterCommit string `json:"rootAfterCommit"` + BlocksInProof []string `json:"blocksInProof"` +} + +func LoadCommitProofFixtures(p string) []CommitProofFixture { + b, err := os.ReadFile(p) + if err != nil { + panic(err) + } + var fixtures []CommitProofFixture + if err := json.Unmarshal(b, &fixtures); err != nil { + panic(err) + } + return fixtures +} + +func (f *CommitProofFixture) RecordCID() cid.Cid { + c, err := cid.Decode(f.LeafValue) + if err != nil { + panic(err) + } + return c +} + +func (f *CommitProofFixture) Tree() mst.Tree { + m := map[string]cid.Cid{} + c := f.RecordCID() + for _, k := range f.Keys { + m[k] = c + } + tree, err := mst.LoadTreeFromMap(m) + if err != nil { + panic(err) + } + return *tree +} + +func (f *CommitProofFixture) Operations() []Operation { + c := f.RecordCID() + ops := []Operation{} + for _, key := range f.Additions { + ops = append(ops, Operation{ + Path: key, + Value: &c, + Prev: nil, + }) + } + for _, key := range f.Deletions { + ops = append(ops, Operation{ + Path: key, + Value: nil, + Prev: &c, + }) + } + return ops +} + +func (f *CommitProofFixture) Test(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + tree := f.Tree() + ops := f.Operations() + + // verify "before" tree CID + before, err := tree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(f.RootBeforeCommit, before.String()) + assert.NoError(tree.Verify()) + + // apply all ops, and generate diff + for _, o := range ops { + _, err := ApplyOp(&tree, o.Path, o.Value) + if err != nil { + t.Fatal(err) + } + } + diffBlocks := blockstore.NewBlockstore(datastore.NewMapDatastore()) + diffRoot, err := tree.WriteDiffBlocks(ctx, diffBlocks) + if err != nil { + t.Fatal(err) + } + assert.Equal(f.RootAfterCommit, diffRoot.String()) + diffTree, err := mst.LoadTreeFromStore(ctx, diffBlocks, *diffRoot) + if err != nil { + t.Fatal(err) + } + assert.NoError(diffTree.Verify()) + + // print out blocks (for development) + /* + akc, err := diffBlocks.AllKeysChan(ctx) + if err != nil { + t.Fatal(err) + } + fmt.Printf("## %s\n", f.Comment) + for c := range akc { + fmt.Println(c) + } + */ + + // verify inclusion of blocks in diff + for _, cstr := range f.BlocksInProof { + c, err := cid.Decode(cstr) + if err != nil { + t.Fatal(err) + } + _, err = diffBlocks.Get(ctx, c) + assert.NoError(err) + } + + // invert all ops + for _, o := range ops { + assert.NoError(CheckOp(diffTree, &o)) + assert.NoError(InvertOp(diffTree, &o)) + } + + // verify "before" tree CID + checkCID, err := diffTree.RootCID() + assert.NoError(err) + assert.Equal(f.RootBeforeCommit, checkCID.String()) +} + +func TestCommitProofFixtures(t *testing.T) { + fixtures := LoadCommitProofFixtures("testdata/commit-proof-fixtures.json") + + for _, f := range fixtures { + f.Test(t) + } +} diff --git a/atproto/repo/mst/cbor_gen.go b/atproto/repo/mst/cbor_gen.go new file mode 100644 index 000000000..3b4bce6d9 --- /dev/null +++ b/atproto/repo/mst/cbor_gen.go @@ -0,0 +1,433 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package mst + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *NodeData) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Entries ([]mst.EntryData) (slice) + if len("e") > 1000000 { + return xerrors.Errorf("Value in field \"e\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("e"))); err != nil { + return err + } + if _, err := cw.WriteString(string("e")); err != nil { + return err + } + + if len(t.Entries) > 8192 { + return xerrors.Errorf("Slice value in field t.Entries was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Entries))); err != nil { + return err + } + for _, v := range t.Entries { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } + + // t.Left (cid.Cid) (struct) + if len("l") > 1000000 { + return xerrors.Errorf("Value in field \"l\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("l"))); err != nil { + return err + } + if _, err := cw.WriteString(string("l")); err != nil { + return err + } + + if t.Left == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Left); err != nil { + return xerrors.Errorf("failed to write cid field t.Left: %w", err) + } + } + + return nil +} + +func (t *NodeData) UnmarshalCBOR(r io.Reader) (err error) { + *t = NodeData{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("NodeData: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 1) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Entries ([]mst.EntryData) (slice) + case "e": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.Entries: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Entries = make([]EntryData, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + if err := t.Entries[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Entries[i]: %w", err) + } + + } + + } + } + // t.Left (cid.Cid) (struct) + case "l": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Left: %w", err) + } + + t.Left = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *EntryData) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.KeySuffix ([]uint8) (slice) + if len("k") > 1000000 { + return xerrors.Errorf("Value in field \"k\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("k"))); err != nil { + return err + } + if _, err := cw.WriteString(string("k")); err != nil { + return err + } + + if len(t.KeySuffix) > 2097152 { + return xerrors.Errorf("Byte array in field t.KeySuffix was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.KeySuffix))); err != nil { + return err + } + + if _, err := cw.Write(t.KeySuffix); err != nil { + return err + } + + // t.PrefixLen (int64) (int64) + if len("p") > 1000000 { + return xerrors.Errorf("Value in field \"p\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("p"))); err != nil { + return err + } + if _, err := cw.WriteString(string("p")); err != nil { + return err + } + + if t.PrefixLen >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.PrefixLen)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.PrefixLen-1)); err != nil { + return err + } + } + + // t.Right (cid.Cid) (struct) + if len("t") > 1000000 { + return xerrors.Errorf("Value in field \"t\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("t"))); err != nil { + return err + } + if _, err := cw.WriteString(string("t")); err != nil { + return err + } + + if t.Right == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Right); err != nil { + return xerrors.Errorf("failed to write cid field t.Right: %w", err) + } + } + + // t.Value (cid.Cid) (struct) + if len("v") > 1000000 { + return xerrors.Errorf("Value in field \"v\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("v"))); err != nil { + return err + } + if _, err := cw.WriteString(string("v")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Value); err != nil { + return xerrors.Errorf("failed to write cid field t.Value: %w", err) + } + + return nil +} + +func (t *EntryData) UnmarshalCBOR(r io.Reader) (err error) { + *t = EntryData{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("EntryData: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 1) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.KeySuffix ([]uint8) (slice) + case "k": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 2097152 { + return fmt.Errorf("t.KeySuffix: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.KeySuffix = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.KeySuffix); err != nil { + return err + } + + // t.PrefixLen (int64) (int64) + case "p": + { + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + var extraI int64 + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative overflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.PrefixLen = int64(extraI) + } + // t.Right (cid.Cid) (struct) + case "t": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Right: %w", err) + } + + t.Right = &c + } + + } + // t.Value (cid.Cid) (struct) + case "v": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Value: %w", err) + } + + t.Value = c + + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} diff --git a/atproto/repo/mst/debug.go b/atproto/repo/mst/debug.go new file mode 100644 index 000000000..90ae54897 --- /dev/null +++ b/atproto/repo/mst/debug.go @@ -0,0 +1,117 @@ +package mst + +import ( + "fmt" + "sort" + + "github.com/ipfs/go-cid" +) + +func debugPrintMap(m map[string]cid.Cid) { + keys := make([]string, len(m)) + i := 0 + for k := range m { + keys[i] = k + i++ + } + sort.Strings(keys) + for _, k := range keys { + fmt.Printf("%s\t%s\n", k, m[k]) + } +} + +// This function is not very well implemented or correct. Should probably switch to Devin's `goat repo mst` code. +func DebugPrintTree(n *Node, depth int) { + if n == nil { + fmt.Printf("EMPTY TREE") + return + } + if depth == 0 { + fmt.Printf("tree root (height=%d)\n", n.Height) + } + for i, e := range n.Entries { + if depth > 0 && i == 0 { + if len(n.Entries) > 1 { + fmt.Printf("┬") + } else { + fmt.Printf("─") + } + } else { + for range depth { + fmt.Printf("│") + } + if i+1 == len(n.Entries) { + fmt.Printf("└") + } else { + fmt.Printf("├") + } + } + if e.IsValue() { + fmt.Printf(" (%d) %s -> %s\n", HeightForKey(e.Key), e.Key, e.Value) + } else if e.IsChild() { + if e.Child != nil { + DebugPrintTree(e.Child, depth+1) + } else { + fmt.Printf("─ (%d; partial) %s\n", n.Height-1, e.ChildCID) + } + } else { + fmt.Printf(" BAD NODE\n") + } + } +} + +func debugCountEntries(n *Node) int { + if n == nil { + return 0 + } + count := 0 + for _, e := range n.Entries { + if e.IsValue() { + count++ + } + if e.IsChild() && e.Child != nil { + count += debugCountEntries(e.Child) + } + } + return count +} + +func debugPrintNodePointers(n *Node) { + if n == nil { + return + } + fmt.Printf("%p %p\n", n, n.Entries) + for _, e := range n.Entries { + if e.IsChild() && e.Child != nil { + debugPrintNodePointers(e.Child) + } + } +} + +func debugPrintChildPointers(n *Node) { + if n == nil { + return + } + for _, e := range n.Entries { + if e.IsChild() && e.Child != nil { + fmt.Printf("CHILD PTR: %p entry: %p\n", e.Child, &e) + debugPrintChildPointers(e.Child) + } + } +} + +func debugSiblingChild(n *Node) error { + lastChild := false + for _, e := range n.Entries { + if e.IsChild() { + if lastChild { + return fmt.Errorf("neighboring children in entries list") + } + lastChild = true + } + if e.IsValue() { + lastChild = false + } + } + return nil +} diff --git a/atproto/repo/mst/doc.go b/atproto/repo/mst/doc.go new file mode 100644 index 000000000..871227cfe --- /dev/null +++ b/atproto/repo/mst/doc.go @@ -0,0 +1,34 @@ +/* +Implementation of the Merkle Search Tree (MST) data structure for atproto. + +## Terminology + +node: any node in the tree. nodes can contain multiple entries. they should never be entirely "empty", unless the entire tree is a single empty node, but they might only contain "child" pointers + +entry: nodes contain multiple entries. these can include both key/CID pairs, or pointers to child nodes. entries are always lexically sorted, with "child" entries pointing to nodes containing (recursively) keys in the appropriate lexical range. there should never be multiple "child" entries adjacent in a single node (they should be merged instead) + +tree: an overall tree of nodes + +## Tricky Bits + +When inserting: + +- the inserted key might be on a "higher" layer than the current top of the tree, in which case new parent tree nodes need to be created +- "parent" or "child" insertions might be multiple layers away from the starting node, with intermediate nodes created +- inserting a "value" entry in a node might require "splitting" a child node, if the key on the current layer would have fallen within the lexical range of the child + +When removing: + +- deleting an entry from a node might result in a "merge" of two child nodes which are no longer "split" +- removing a "value" entry from the top of the tree might make it a simple pointer down to a child. in this case the top of the tree should be "trimmed" (this might involve multiple layers of trimming) + +When inverting operations: + +- need additional "proof" blocks to invert deletions. basically need the proof blocks for any keys (at any layer) directly adjacent to the deleted block +- if an entry is removed from the top of a partial tree and results in "trimming", and the child node is not available, the overall tree root CID might still be known + +## Hacking + +Be careful with go slices. Need to avoid creating multiple references (slices) of the same underlying array, which can lead to "mutation at a distance" in ways that are hard to debug. +*/ +package mst diff --git a/atproto/repo/mst/encoding.go b/atproto/repo/mst/encoding.go new file mode 100644 index 000000000..ff35f5570 --- /dev/null +++ b/atproto/repo/mst/encoding.go @@ -0,0 +1,234 @@ +package mst + +import ( + "bytes" + "context" + "fmt" + "io" + + bf "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + ipld "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multihash" +) + +// CBOR serialization struct for a MST tree node. MST tree node as gets serialized to CBOR. Note that the CBOR fields are all single-character. +type NodeData struct { + Left *cid.Cid `cborgen:"l"` // [nullable] pointer to lower-level subtree to the "left" of this path/key + Entries []EntryData `cborgen:"e"` // ordered list of entries at this node +} + +// CBOR serialization struct for a single entry within a `NodeData` entry list. +type EntryData struct { + PrefixLen int64 `cborgen:"p"` // count of characters shared with previous path/key in tree + KeySuffix []byte `cborgen:"k"` // remaining part of path/key (appended to "previous key") + Value cid.Cid `cborgen:"v"` // CID pointer at this path/key + Right *cid.Cid `cborgen:"t"` // [nullable] pointer to lower-level subtree to the "right" of this path/key entry +} + +// Encodes a single `NodeData` struct as CBOR bytes. Does not recursively encode or update children. +func (d *NodeData) Bytes() ([]byte, *cid.Cid, error) { + buf := new(bytes.Buffer) + if err := d.MarshalCBOR(buf); err != nil { + return nil, nil, err + } + b := buf.Bytes() + builder := cid.NewPrefixV1(cid.DagCBOR, multihash.SHA2_256) + c, err := builder.Sum(b) + if err != nil { + return nil, nil, err + } + return b, &c, nil +} + +// Parses CBOR bytes in to `NodeData` struct +func NodeDataFromCBOR(r io.Reader) (*NodeData, error) { + var nd NodeData + if err := nd.UnmarshalCBOR(r); err != nil { + return nil, err + } + // TODO: verify CID type, and "non-empty" here? + return &nd, nil +} + +// Transforms `Node` struct to `NodeData`, which is the format used for encoding to CBOR. +// +// Will panic if any entries are missing a CID (must compute those first) +func (n *Node) NodeData() NodeData { + d := NodeData{ + Left: nil, + Entries: []EntryData{}, // TODO perf: pre-allocate an array + } + + prevKey := []byte{} + for i, e := range n.Entries { + if i == 0 && e.IsChild() { + d.Left = e.ChildCID + continue + } + if e.IsChild() { + if len(d.Entries) == 0 { + panic("malformed tree node") // TODO: return error? + } + d.Entries[len(d.Entries)-1].Right = e.ChildCID + } + if e.IsValue() { + idx := int64(CountPrefixLen(prevKey, e.Key)) + d.Entries = append(d.Entries, EntryData{ + PrefixLen: idx, + KeySuffix: e.Key[idx:], + Value: *e.Value, + Right: nil, + }) + prevKey = e.Key + } + } + return d +} + +// Tansforms an encoded `NodeData` to `Node` data structure format. +// +// c: optional CID argument for the CID of the CBOR representation of the NodeData +func (d *NodeData) Node(c *cid.Cid) Node { + height := -1 + n := Node{ + CID: c, + Dirty: c == nil, + Entries: []NodeEntry{}, // TODO: pre-allocate + } + + if d.Left != nil { + n.Entries = append(n.Entries, NodeEntry{ChildCID: d.Left}) + } + + var prevKey []byte + for _, e := range d.Entries { + // TODO perf: pre-allocate + key := []byte{} + key = append(key, prevKey[:e.PrefixLen]...) + key = append(key, e.KeySuffix...) + n.Entries = append(n.Entries, NodeEntry{ + Key: key, + Value: &e.Value, + }) + prevKey = key + if height < 0 { + height = HeightForKey(key) + } + + if e.Right != nil { + n.Entries = append(n.Entries, NodeEntry{ + ChildCID: e.Right, + }) + } + } + + // TODO: height doesn't get set properly if this is an intermediate node; we rely on `EnsureHeights` getting called to fix that + n.Height = height + return n +} + +// TODO: this feels like a hack, and easy to forget +func (n *Node) ensureHeights() { + if n.Height <= 0 { + return + } + for _, e := range n.Entries { + if e.Child != nil { + if n.Height > 0 && e.Child.Height < 0 { + e.Child.Height = n.Height - 1 + } + e.Child.ensureHeights() + } + } +} + +// Recursively encodes sub-tree, optionally writing to blockstore. Returns root CID. +// +// This method will not error if tree is partial. +// +// bs: is an optional blockstore; if it is nil, blocks will not be written. +// onlyDirty: is an optional blockstore; if it is nil, blocks will not be written. +func (n *Node) writeBlocks(ctx context.Context, bs blockstore.Blockstore, onlyDirty bool) (*cid.Cid, error) { + if n == nil || n.Stub { + return nil, fmt.Errorf("%w: nil tree node", ErrInvalidTree) + } + if onlyDirty && !n.Dirty && n.CID != nil { + return n.CID, nil + } + + // walk all children first + for i, e := range n.Entries { + if e.IsValue() && e.Dirty { + // TODO: should we actually clear this here? + e.Dirty = false + } + if !e.IsChild() { + continue + } + if e.Child != nil && (e.Dirty || e.Child.Dirty || !onlyDirty) { + cc, err := e.Child.writeBlocks(ctx, bs, onlyDirty) + if err != nil { + return nil, err + } + n.Entries[i].ChildCID = cc + n.Entries[i].Dirty = false + } + } + + // compute this block + nd := n.NodeData() + b, c, err := nd.Bytes() + if err != nil { + return nil, err + } + + n.CID = c + n.Dirty = false + + if bs != nil { + blk, err := bf.NewBlockWithCid(b, *c) + if err != nil { + return nil, err + } + if err := bs.Put(ctx, blk); err != nil { + return nil, err + } + } + return c, nil +} + +func loadNodeFromStore(ctx context.Context, bs MSTBlockSource, ref cid.Cid) (*Node, error) { + block, err := bs.Get(ctx, ref) + if err != nil { + return nil, err + } + + nd, err := NodeDataFromCBOR(bytes.NewReader(block.RawData())) + if err != nil { + return nil, err + } + + n := nd.Node(&ref) + + for i, e := range n.Entries { + if e.IsChild() { + child, err := loadNodeFromStore(ctx, bs, *e.ChildCID) + if err != nil && ipld.IsNotFound(err) { + // allow "partial" trees + continue + } + if err != nil { + return nil, err + } + n.Entries[i].Child = child + // NOTE: this is kind of a hack + if n.Height == -1 && child.Height >= 0 { + n.Height = child.Height + 1 + } + } + } + + return &n, nil +} diff --git a/atproto/repo/mst/mst_interop_test.go b/atproto/repo/mst/mst_interop_test.go new file mode 100644 index 000000000..72b66e897 --- /dev/null +++ b/atproto/repo/mst/mst_interop_test.go @@ -0,0 +1,307 @@ +// This file contains tests which are the same across language implementations. +// AKA, if you update this file, you should probably update the corresponding +// file in atproto repo (typescript) +package mst + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/ipfs/go-cid" +) + +func mapToCidMapDecode(t *testing.T, a map[string]string) map[string]cid.Cid { + out := make(map[string]cid.Cid) + for k, v := range a { + c, err := cid.Decode(v) + if err != nil { + t.Fatal(err) + } + out[k] = c + } + return out +} + +func mapToTreeRootCidString(t *testing.T, m map[string]string) string { + + tree, err := LoadTreeFromMap(mapToCidMapDecode(t, m)) + if err != nil { + t.Fatal(err) + } + + c, err := tree.RootCID() + if err != nil { + t.Fatal(err) + } + + return c.String() +} + +// TODO: TestAllowedKeys + +func TestManualNode(t *testing.T) { + assert := assert.New(t) + + cid1, err := cid.Decode("bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454") + if err != nil { + t.Fatal(err) + } + + simple_nd := NodeData{ + Left: nil, + Entries: []EntryData{ + { + PrefixLen: 0, + KeySuffix: []byte("com.example.record/3jqfcqzm3fo2j"), + Value: cid1, + Right: nil, + }, + }, + } + n := simple_nd.Node(nil) + assert.Equal(simple_nd, n.NodeData()) + + mcid, err := n.writeBlocks(context.Background(), nil, true) + if err != nil { + t.Fatal(err) + } + assert.NoError(err) + assert.Equal("bafyreibj4lsc3aqnrvphp5xmrnfoorvru4wynt6lwidqbm2623a6tatzdu", mcid.String()) +} + +func TestInteropKnownMaps(t *testing.T) { + assert := assert.New(t) + + cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" + + // empty map + emptyMap := map[string]string{} + assert.Equal("bafyreie5737gdxlw5i64vzichcalba3z2v5n6icifvx5xytvske7mr3hpm", mapToTreeRootCidString(t, emptyMap)) + + // no depth, single entry + trivialMap := map[string]string{ + "com.example.record/3jqfcqzm3fo2j": cid1str, + } + assert.Equal("bafyreibj4lsc3aqnrvphp5xmrnfoorvru4wynt6lwidqbm2623a6tatzdu", mapToTreeRootCidString(t, trivialMap)) + + // single layer=2 entry + singlelayer2Map := map[string]string{ + "com.example.record/3jqfcqzm3fx2j": cid1str, + } + assert.Equal("bafyreih7wfei65pxzhauoibu3ls7jgmkju4bspy4t2ha2qdjnzqvoy33ai", mapToTreeRootCidString(t, singlelayer2Map)) + + // pretty simple, but with some depth + simpleMap := map[string]string{ + "com.example.record/3jqfcqzm3fp2j": cid1str, + "com.example.record/3jqfcqzm3fr2j": cid1str, + "com.example.record/3jqfcqzm3fs2j": cid1str, + "com.example.record/3jqfcqzm3ft2j": cid1str, + "com.example.record/3jqfcqzm4fc2j": cid1str, + } + assert.Equal("bafyreicmahysq4n6wfuxo522m6dpiy7z7qzym3dzs756t5n7nfdgccwq7m", mapToTreeRootCidString(t, simpleMap)) +} + +func TestInteropKnownMapsTricky(t *testing.T) { + t.Skip("TODO: these are currently disallowed in typescript implementation") + assert := assert.New(t) + + cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" + + // include several known edge cases + trickyMap := map[string]string{ + "": cid1str, + "jalapeño": cid1str, + "coöperative": cid1str, + "coüperative": cid1str, + "abc\x00": cid1str, + } + assert.Equal("bafyreiecb33zh7r2sc3k2wthm6exwzfktof63kmajeildktqc25xj6qzx4", mapToTreeRootCidString(t, trickyMap)) +} + +// "trims top of tree on delete" +func TestInteropEdgeCasesTrimTop(t *testing.T) { + assert := assert.New(t) + + cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" + l1root := "bafyreifnqrwbk6ffmyaz5qtujqrzf5qmxf7cbxvgzktl4e3gabuxbtatv4" + l0root := "bafyreie4kjuxbwkhzg2i5dljaswcroeih4dgiqq6pazcmunwt2byd725vi" + + trimMap := map[string]string{ + "com.example.record/3jqfcqzm3fn2j": cid1str, // level 0 + "com.example.record/3jqfcqzm3fo2j": cid1str, // level 0 + "com.example.record/3jqfcqzm3fp2j": cid1str, // level 0 + "com.example.record/3jqfcqzm3fs2j": cid1str, // level 0 + "com.example.record/3jqfcqzm3ft2j": cid1str, // level 0 + "com.example.record/3jqfcqzm3fu2j": cid1str, // level 1 + } + trimTree, err := LoadTreeFromMap(mapToCidMapDecode(t, trimMap)) + if err != nil { + t.Fatal(err) + } + trimBefore, err := trimTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(1, trimTree.Root.Height) + assert.Equal(l1root, trimBefore.String()) + + _, err = trimTree.Remove([]byte("com.example.record/3jqfcqzm3fs2j")) // level 1 + if err != nil { + t.Fatal(err) + } + trimAfter, err := trimTree.RootCID() + if err != nil { + t.Fatal(err) + } + //fmt.Printf("%#v\n", trimTree) + assert.Equal(0, trimTree.Root.Height) + assert.Equal(l0root, trimAfter.String()) +} + +func TestInteropEdgeCasesInsertion(t *testing.T) { + assert := assert.New(t) + + cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" + cid1, err := cid.Decode(cid1str) + if err != nil { + t.Fatal(err) + } + + // "handles insertion that splits two layers down" + l1root := "bafyreiettyludka6fpgp33stwxfuwhkzlur6chs4d2v4nkmq2j3ogpdjem" + l2root := "bafyreid2x5eqs4w4qxvc5jiwda4cien3gw2q6cshofxwnvv7iucrmfohpm" + insertionMap := map[string]string{ + "com.example.record/3jqfcqzm3fo2j": cid1str, // A; level 0 + "com.example.record/3jqfcqzm3fp2j": cid1str, // B; level 0 + "com.example.record/3jqfcqzm3fr2j": cid1str, // C; level 0 + "com.example.record/3jqfcqzm3fs2j": cid1str, // D; level 1 + "com.example.record/3jqfcqzm3ft2j": cid1str, // E; level 0 + "com.example.record/3jqfcqzm3fz2j": cid1str, // G; level 0 + "com.example.record/3jqfcqzm4fc2j": cid1str, // H; level 0 + "com.example.record/3jqfcqzm4fd2j": cid1str, // I; level 1 + "com.example.record/3jqfcqzm4ff2j": cid1str, // J; level 0 + "com.example.record/3jqfcqzm4fg2j": cid1str, // K; level 0 + "com.example.record/3jqfcqzm4fh2j": cid1str, // L; level 0 + } + insertionTree, err := LoadTreeFromMap(mapToCidMapDecode(t, insertionMap)) + if err != nil { + t.Fatal(err) + } + insertionBefore, err := insertionTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(1, insertionTree.Root.Height) + assert.Equal(l1root, insertionBefore.String()) + + // insert F, which will push E out of the node with G+H to a new node under D + _, err = insertionTree.Insert([]byte("com.example.record/3jqfcqzm3fx2j"), cid1) // F; level 2 + if err != nil { + t.Fatal(err) + } + insertionAfter, err := insertionTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(2, insertionTree.Root.Height) + assert.Equal(l2root, insertionAfter.String()) + + // remove F, which should push E back over with G+H + _, err = insertionTree.Remove([]byte("com.example.record/3jqfcqzm3fx2j")) // F; level 2 + if err != nil { + t.Fatal(err) + } + insertionFinal, err := insertionTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(1, insertionTree.Root.Height) + assert.Equal(l1root, insertionFinal.String()) +} + +// "handles new layers that are two higher than existing" +func TestInteropEdgeCasesHigher(t *testing.T) { + assert := assert.New(t) + + cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" + cid1, err := cid.Decode(cid1str) + if err != nil { + t.Fatal(err) + } + + l0root := "bafyreidfcktqnfmykz2ps3dbul35pepleq7kvv526g47xahuz3rqtptmky" + l2root := "bafyreiavxaxdz7o7rbvr3zg2liox2yww46t7g6hkehx4i4h3lwudly7dhy" + l2root2 := "bafyreig4jv3vuajbsybhyvb7gggvpwh2zszwfyttjrj6qwvcsp24h6popu" + higherMap := map[string]string{ + "com.example.record/3jqfcqzm3ft2j": cid1str, // A; level 0 + "com.example.record/3jqfcqzm3fz2j": cid1str, // C; level 0 + } + higherTree, err := LoadTreeFromMap(mapToCidMapDecode(t, higherMap)) + if err != nil { + t.Fatal(err) + } + higherBefore, err := higherTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(0, higherTree.Root.Height) + assert.Equal(l0root, higherBefore.String()) + + // insert B, which is two levels above + _, err = higherTree.Insert([]byte("com.example.record/3jqfcqzm3fx2j"), cid1) // B; level 2 + if err != nil { + t.Fatal(err) + } + higherAfter, err := higherTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(l2root, higherAfter.String()) + //debugPrintTree(higherTree, 0) + + // remove B + _, err = higherTree.Remove([]byte("com.example.record/3jqfcqzm3fx2j")) // B; level 2 + if err != nil { + t.Fatal(err) + } + higherAgain, err := higherTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(0, higherTree.Root.Height) + assert.Equal(l0root, higherAgain.String()) + + // insert B (level=2) and D (level=1) + _, err = higherTree.Insert([]byte("com.example.record/3jqfcqzm3fx2j"), cid1) // B; level 2 + if err != nil { + t.Fatal(err) + } + _, err = higherTree.Insert([]byte("com.example.record/3jqfcqzm4fd2j"), cid1) // D; level 1 + if err != nil { + t.Fatal(err) + } + higherYetAgain, err := higherTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(2, higherTree.Root.Height) + assert.Equal(l2root2, higherYetAgain.String()) + assert.NoError(higherTree.Verify()) + //debugPrintTree(higherTree, 0) + + // remove D + _, err = higherTree.Remove([]byte("com.example.record/3jqfcqzm4fd2j")) // D; level 1 + if err != nil { + t.Fatal(err) + } + higherFinal, err := higherTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(2, higherTree.Root.Height) + assert.Equal(l2root, higherFinal.String()) + assert.NoError(higherTree.Verify()) + //debugPrintTree(higherTree, 0) +} diff --git a/atproto/repo/mst/mst_test.go b/atproto/repo/mst/mst_test.go new file mode 100644 index 000000000..266d505d4 --- /dev/null +++ b/atproto/repo/mst/mst_test.go @@ -0,0 +1,276 @@ +package mst + +import ( + "bytes" + "encoding/hex" + "math/rand" + "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" +) + +func TestBasicMST(t *testing.T) { + assert := assert.New(t) + + c2, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222") + c3, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu333333333") + assert.NotEmpty(c2) + assert.NotEmpty(c3) + tree := NewEmptyTree() + + prev, err := tree.Insert([]byte("abc"), c2) + assert.NoError(err) + assert.Empty(prev) + + assert.Equal(1, len(tree.Root.Entries)) + + val, err := tree.Get([]byte("abc")) + assert.NoError(err) + assert.Equal(c2, *val) + + val, err = tree.Get([]byte("xyz")) + assert.NoError(err) + assert.Empty(val) + + prev, err = tree.Insert([]byte("abc"), c3) + assert.NoError(err) + assert.NotEmpty(prev) + assert.Equal(&c2, prev) + + val, err = tree.Get([]byte("abc")) + assert.NoError(err) + assert.Equal(&c3, val) + + prev, err = tree.Insert([]byte("aaa"), c2) + assert.NoError(err) + assert.Empty(prev) + + prev, err = tree.Insert([]byte("zzz"), c3) + assert.NoError(err) + assert.Empty(prev) + + val, err = tree.Get([]byte("zzz")) + assert.NoError(err) + assert.Equal(&c3, val) + + m := make(map[string]cid.Cid) + assert.NoError(tree.WriteToMap(m)) + //fmt.Println("-----") + //debugPrintMap(m) + //fmt.Println("-----") + //debugPrintTree(tree, 0) + + prev, err = tree.Remove([]byte("abc")) + assert.NoError(err) + assert.NotEmpty(prev) + assert.Equal(&c3, prev) + + assert.NoError(tree.Verify()) + +} + +func TestKeyLimits(t *testing.T) { + assert := assert.New(t) + + var err error + tree := NewEmptyTree() + c2, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222") + + emptyKey := []byte{} + _, err = tree.Get(emptyKey) + assert.Error(err) + _, err = tree.Remove(emptyKey) + assert.Error(err) + _, err = tree.Insert(emptyKey, c2) + assert.Error(err) + + bigKey := bytes.Repeat([]byte{'a'}, 3000) + _, err = tree.Get(bigKey) + assert.Error(err) + _, err = tree.Remove(bigKey) + assert.Error(err) + _, err = tree.Insert(bigKey, c2) + assert.Error(err) +} + +func TestBasicMap(t *testing.T) { + assert := assert.New(t) + + c2, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222") + c3, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu333333333") + assert.NotEmpty(c2) + assert.NotEmpty(c3) + + inMap := map[string]cid.Cid{ + "a": c2, + "b": c2, + "c": c2, + "d": c3, + "e": c3, + "f": c3, + "g": c3, + "h": c3, + "i": c3, + } + + tree, err := LoadTreeFromMap(inMap) + assert.NoError(err) + + //fmt.Println("-----") + //debugPrintTree(tree, 0) + assert.NoError(tree.Verify()) + + outMap := make(map[string]cid.Cid, len(inMap)) + err = tree.WriteToMap(outMap) + assert.NoError(err) + assert.Equal(inMap, outMap) +} + +func randomCid() cid.Cid { + buf := make([]byte, 32) + rand.Read(buf) + c, err := cid.NewPrefixV1(cid.DagCBOR, multihash.SHA2_256).Sum(buf) + if err != nil { + panic(err) + } + return c +} + +func randomStr() string { + buf := make([]byte, 16) + rand.Read(buf) + return hex.EncodeToString(buf) +} + +func TestRandomTree(t *testing.T) { + assert := assert.New(t) + + size := 200 + + inMap := make(map[string]cid.Cid, size) + outMap := make(map[string]cid.Cid, size) + + for range size { + k := randomStr() + // ensure key is not already in the random set + for { + _, ok := inMap[k] + if !ok { + break + } + k = randomStr() + } + inMap[k] = randomCid() + } + + tree, err := LoadTreeFromMap(inMap) + assert.NoError(err) + + //fmt.Println("-----") + //debugPrintTree(tree, 0) + assert.NoError(tree.Verify()) + assert.Equal(size, debugCountEntries(tree.Root)) + + err = tree.WriteToMap(outMap) + assert.NoError(err) + assert.Equal(len(inMap), len(outMap)) + assert.Equal(inMap, outMap) + + mapKeys := make([]string, len(inMap)) + i := 0 + for k := range inMap { + mapKeys[i] = k + i++ + } + rand.Shuffle(len(mapKeys), func(i, j int) { + mapKeys[i], mapKeys[j] = mapKeys[j], mapKeys[i] + }) + + // test gets + for _, k := range mapKeys { + val, err := tree.Get([]byte(k)) + assert.NoError(err) + assert.Equal(inMap[k], *val) + } + + // finally, removals + var val *cid.Cid + for _, k := range mapKeys { + val, err = tree.Remove([]byte(k)) + assert.NoError(err) + assert.NotNil(val) + if err != nil { + break + } + err = tree.Verify() + assert.NoError(err) + if err != nil { + break + } + } +} + +func TestRandomUntilError(t *testing.T) { + assert := assert.New(t) + var err error + var prev *cid.Cid + + size := 200 + + tree := NewEmptyTree() + count := 0 + //fmt.Println("-----") + for range size { + key := []byte(randomStr()) + val := randomCid() + //fmt.Printf("%s %s\n", key, val) + prev, err = tree.Insert(key, val) + assert.NoError(err) + if prev == nil { + count++ + } + + assert.Equal(count, debugCountEntries(tree.Root)) + err = tree.Verify() + assert.NoError(err) + if err != nil || count != debugCountEntries(tree.Root) { + //fmt.Println("-----") + //debugPrintTree(tree, 0) + break + } + } +} + +func TestBrokenCaseOne(t *testing.T) { + assert := assert.New(t) + var err error + + entries := [][]string{ + {"1ea173efefa4", "bafyreibey6qzs7vb4wzlzfo7flflevl7qstzaggooiqivuexb6snapadq4"}, + {"bed5c5789108", "bafyreifoxw552rsnuoargsfilhwmhprxr6qyzjmbtgjzmboii4x4mk4aoi"}, + {"340b57a94d4c", "bafyreigarcm3fvnekjml6vmm5dyg46qnfkpc2lhghnh2wvntwvbrvxzq7q"}, + {"8d37e30d3d29", "bafyreifdgiz7dmgng4aebiw5m6w4cypiiar2edgtkkfg47o3pniir3pxve"}, + {"ee4b5efda333", "bafyreiho7qtewg7fm7egxe2ectkm2ykqygakph3nt4rrlp5mxwkvdwckk4"}, + {"1180aeeadc01", "bafyreifqhtleufnxv2nkwehoa5lgmilwgkfqvlpkwbalvka6m6675ewkhu"}, + {"c368b6b55998", "bafyreial4xepr5wnhetxnkmylmipdmjybxsgf74becdi74olmzb5w5gpiq"}, + {"b948d2e0fc76", "bafyreiaefdmlyfjf4qovfyn22zbpw57wu667jtrvavogfxr7drewx4u24y"}, + {"93c53d491ffd", "bafyreie2nxdmjsy6k6lendnsy7bzyufj7j37l42ymquwmpuzsauraqsibq"}, + {"54ef0958a374", "bafyreigbnjxc7wbxgqxs2n2djjmlxnuf222gdiq4jgdtkse4yn67v5crq4"}, + } + + tree := NewEmptyTree() + for _, row := range entries { + val, _ := cid.Decode(row[1]) + _, err = tree.Insert([]byte(row[0]), val) + assert.NoError(err) + } + + //fmt.Println("-----") + //debugPrintNodePointers(tree) + //debugPrintChildPointers(tree) + //debugPrintTree(tree, 0) + assert.Equal(len(entries), debugCountEntries(tree.Root)) + assert.NoError(tree.Verify()) +} diff --git a/atproto/repo/mst/node.go b/atproto/repo/mst/node.go new file mode 100644 index 000000000..91c2e7f57 --- /dev/null +++ b/atproto/repo/mst/node.go @@ -0,0 +1,352 @@ +package mst + +import ( + "bytes" + "fmt" + + "github.com/ipfs/go-cid" +) + +// Represents a node in a Merkle Search Tree (MST). If this is the "root" or "top" of the tree, it effectively is the tree itself. +// +// Trees may be "partial" if they contain references to child nodes by CID, but not pointers to `Node` representations. +type Node struct { + // array of key/value pairs and pointers to child nodes. entry arrays must always be in correct/valid order at any point in time: sorted by 'key', and at most one 'pointer' entry between 'value' entries. + Entries []NodeEntry + // "height" or "layer" of MST tree this node is at (with zero at the "bottom" and root/top of tree the "highest") + Height int + // if true, the cached CID of this node is out of date + Dirty bool + // optionally, the last computed CID of this Node (when expressed as NodeData) + CID *cid.Cid + // if true, this is an empty/incomplete node which just represents the CID of the tree. only used as part of MST inversion + Stub bool +} + +// Represents an entry in an MST `Node`, which could either be a direct path/value entry, or a pointer do a child tree node. Note that these are *not* one-to-one with `EntryData`. +// +// Either the Key and Value fields should be non-zero; or the Child and/or ChildCID field should be non-zero. +// If ChildCID is present, but Child is not, then this is part of a "partial" tree. +type NodeEntry struct { + Key []byte + Value *cid.Cid + ChildCID *cid.Cid + Child *Node + + // tracks whether anything about this entry has changed since `Node` CID was computed + Dirty bool +} + +func (n *Node) IsEmpty() bool { + return len(n.Entries) == 0 +} + +// Checks if the sub-tree (this node, or any children, recursively) contains any CID references to nodes which are not present. +func (n *Node) IsPartial() bool { + if n.Stub { + return true + } + for _, e := range n.Entries { + if e.ChildCID != nil && e.Child == nil { + return true + } + if e.Child != nil && e.Child.IsPartial() { + return true + } + } + return false +} + +// Returns true if this entry is a key/value at the current node +func (e *NodeEntry) IsValue() bool { + if len(e.Key) > 0 && e.Value != nil { + return true + } + return false +} + +// Returns true if this entry points to a node on a lower level +func (e *NodeEntry) IsChild() bool { + if e.Child != nil || e.ChildCID != nil { + return true + } + return false +} + +// creates a deep/recursive copy of the sub-tree +func (n *Node) deepCopy() *Node { + out := Node{ + Entries: make([]NodeEntry, len(n.Entries)), + Height: n.Height, + Dirty: n.Dirty, + Stub: n.Stub, + CID: n.CID, + } + for i, e := range n.Entries { + out.Entries[i] = NodeEntry{ + Key: e.Key, + Value: e.Value, + ChildCID: e.ChildCID, + Dirty: e.Dirty, + } + if e.Child != nil { + out.Entries[i].Child = e.Child.deepCopy() + } + } + return &out +} + +// Looks for a "value" entry in the node with the exact key. +// Returns entry index if a matching entry is found; or -1 if not found +func (n *Node) findExistingEntry(key []byte) int { + for i, e := range n.Entries { + // TODO perf: could skip early if e.Key is lower + if e.IsValue() && bytes.Equal(key, e.Key) { + return i + } + } + return -1 +} + +// Looks for a "child" entry which the key would live under. +// +// Returns -1 if not found. +func (n *Node) findExistingChild(key []byte) int { + idx := -1 + for i, e := range n.Entries { + if e.IsChild() { + idx = i + continue + } + if e.IsValue() { + if bytes.Compare(key, e.Key) <= 0 { + break + } + idx = -1 + } + } + return idx +} + +// Determines index where a new entry (child or value) would be inserted, relevant to the given key. +// +// If the key would "split" an existing child entry, the index of that entry is returned, and a flag set +// +// If the entry would be appended, then the index returned will be one higher that the current largest index. +func (n *Node) findInsertionIndex(key []byte) (idx int, split bool, retErr error) { + if n.Stub { + return -1, false, fmt.Errorf("partial MST, can't determine insertion order") + } + for i, e := range n.Entries { + if e.IsValue() { + if bytes.Compare(key, e.Key) < 0 { + return i, false, nil + } + } + if e.IsChild() { + // first, see if there is a next entry as a value which this key would be after; if so we can skip checking this child + if i+1 < len(n.Entries) { + next := n.Entries[i+1] + if next.IsValue() && bytes.Compare(key, next.Key) > 0 { + continue + } + } + if e.Child == nil { + return -1, false, fmt.Errorf("partial MST, can't determine insertion order") + } + order, err := e.Child.compareKey(key, false) + if err != nil { + return -1, false, err + } + if order < 0 { + // key comes before this entire child sub-tree + return i, false, nil + } + if order > 0 { + // key comes after this entire child sub-tree + continue + } + // key falls inside this child sub-tree + return i, true, nil + } + } + + // would need to be appended after + return len(n.Entries), false, nil +} + +// Compares a provided `key` against the overall range of keys represented by a `Node`. Returns -1 if the key sorts lower than all keys (recursively) covered by the Node; 1 if higher, and 0 if the key falls within Node's key range. +// +// If the `markDirty` flag is true, then this method will set the Dirty flag on this node, and any child nodes which were needed to "prove" the key order. This can be used to mark nodes for inclusion in invertible MST diffs. +func (n *Node) compareKey(key []byte, markDirty bool) (int, error) { + if n.Stub { + return -1, ErrPartialTree + } + if n.IsEmpty() { + // TODO: should we actually return 0 in this case? + return 0, fmt.Errorf("can't determine key range of empty MST node") + } + if markDirty { + n.Dirty = true + } + // check if lower than this entire node + e := n.Entries[0] + if e.IsValue() && bytes.Compare(key, e.Key) < 0 { + return -1, nil + } + // check if higher than this entire node + e = n.Entries[len(n.Entries)-1] + if e.IsValue() && bytes.Compare(key, e.Key) > 0 { + return 1, nil + } + for i, e := range n.Entries { + if e.IsValue() && bytes.Compare(key, e.Key) < 0 { + // we don't need to recurse/iterate further + return 0, nil + } + if e.IsChild() { + // first, see if there is a next entry as a value which this key would be after; if so we can skip checking this child + if i+1 < len(n.Entries) { + next := n.Entries[i+1] + if next.IsValue() && bytes.Compare(key, next.Key) > 0 { + continue + } + } + if e.Child == nil { + return 0, fmt.Errorf("%w: can't compare key order recursively", ErrPartialTree) + } + order, err := e.Child.compareKey(key, markDirty) + if err != nil { + return 0, err + } + // lower than entire node + if i == 0 && order < 0 { + return -1, nil + } + // higher than entire node + if i == len(n.Entries)-1 && order > 0 { + return 1, nil + } + return 0, nil + } + } + return 0, nil +} + +// helper to mark nodes as "dirty" if they are needed to "prove" something about the key. used to generate invertable operation diffs. +func proveMutation(n *Node, key []byte) error { + for i, e := range n.Entries { + if e.IsValue() { + if bytes.Compare(key, e.Key) < 0 { + return nil + } + } + if e.IsChild() { + // first, see if there is a next entry as a value which this key would be after; if so we can skip checking this child + if i+1 < len(n.Entries) { + next := n.Entries[i+1] + if next.IsValue() && bytes.Compare(key, next.Key) > 0 { + continue + } + } + if e.Child == nil { + return fmt.Errorf("can't prove mutation: %w", ErrPartialTree) + } + order, err := e.Child.compareKey(key, true) + if err != nil { + return err + } + if order > 0 { + // key comes after this entire child sub-tree + continue + } + if order < 0 { + return nil + } + // key falls inside this child sub-tree + return proveMutation(e.Child, key) + } + } + return nil +} + +// helper function, mostly for testing or development, which redusively inserts key/CID pairs into a `map[string]cid.Cid +func (n *Node) writeToMap(m map[string]cid.Cid) error { + if m == nil { + return fmt.Errorf("un-initialized map as an argument") + } + if n == nil { + return fmt.Errorf("nil tree pointer") + } + for _, e := range n.Entries { + if e.IsValue() { + m[string(e.Key)] = *e.Value + } + if e.Child != nil { + if err := e.Child.writeToMap(m); err != nil { + return fmt.Errorf("failed to export MST structure as map: %w", err) + } + } + } + return nil +} + +func (n *Node) walk(f func(key []byte, val cid.Cid) error) error { + if n == nil { + return fmt.Errorf("nil tree pointer") + } + for _, e := range n.Entries { + if e.IsValue() { + if err := f(e.Key, *e.Value); err != nil { + return err + } + } + if e.Child != nil { + if err := e.Child.walk(f); err != nil { + return err + } + } + } + return nil +} + +// Reads the value (CID) corresponding to the key. If key is not in the tree, returns (nil, nil). +// +// n: Node at top of sub-tree to operate on. Must not be nil. +// key: key or path being inserted. must not be empty/nil +// height: tree height corresponding to key. if a negative value is provided, will be computed; use -1 instead of 0 if height is not known +func (n *Node) getCID(key []byte, height int) (*cid.Cid, error) { + if n.Stub { + return nil, ErrPartialTree + } + if height < 0 { + height = HeightForKey(key) + } + + if height > n.Height { + // key from a higher layer; key was not in tree + return nil, nil + } + + if height < n.Height { + // look for a child node + idx := n.findExistingChild(key) + if idx >= 0 { + if n.Entries[idx].Child == nil { + return nil, fmt.Errorf("could not search for key: %w", ErrPartialTree) + } + return n.Entries[idx].Child.getCID(key, height) + } + // otherwise, not found + return nil, nil + } + + // search at this height + idx := n.findExistingEntry(key) + if idx >= 0 { + return n.Entries[idx].Value, nil + } + + // not found + return nil, nil +} diff --git a/atproto/repo/mst/node_insert.go b/atproto/repo/mst/node_insert.go new file mode 100644 index 000000000..54852cecd --- /dev/null +++ b/atproto/repo/mst/node_insert.go @@ -0,0 +1,236 @@ +package mst + +import ( + "errors" + "fmt" + "slices" + + "github.com/ipfs/go-cid" +) + +// Adds a key/CID entry to a sub-tree defined by a Node. If a previous value existed, returns it. +// +// If the insert is a no-op (the key already existed with exact value), then the operation is a no-op, the tree is not marked dirty, and the val is returned as the 'prev' value. +// +// n: Node at top of sub-tree to operate on +// key: key or path being inserted. must not be empty/nil +// val: CID value being inserted +// height: tree height to insert at, derived from key. if a negative value is provided, will be computed; use -1 instead of 0 if height is not known +func (n *Node) insert(key []byte, val cid.Cid, height int) (*Node, *cid.Cid, error) { + if n.Stub { + return nil, nil, ErrPartialTree + } + if height < 0 { + height = HeightForKey(key) + } + + if n == nil { + return nil, nil, fmt.Errorf("operating on nil tree/node") + } + + for height > n.Height { + // if the new key is higher in the tree; will need to add a parent node, which may involve splitting this current node + return n.insertParent(key, val, height) + } + + // if key is lower on the tree, we need to descend first + if height < n.Height { + return n.insertChild(key, val, height) + } + + // look for existing key + idx := n.findExistingEntry(key) + if idx >= 0 { + e := n.Entries[idx] + if *e.Value == val { + // same value already exists; no-op + return n, &val, nil + } + // update operation + prev := e.Value + n.Entries[idx].Value = &val + n.Entries[idx].Dirty = true + n.Dirty = true + return n, prev, nil + } + + // insert new entry to this node + idx, split, err := n.findInsertionIndex(key) + if err != nil { + return nil, nil, err + } + n.Dirty = true + newEntry := NodeEntry{ + Key: key, + Value: &val, + Dirty: true, + } + + // include "covering" proof for this operation + if err := proveMutation(n, key); err != nil && !errors.Is(err, ErrPartialTree) { + return nil, nil, err + } + + if !split { + // TODO: is this really necessary? or can we just slices.Insert beyond the end of a slice? + if idx >= len(n.Entries) { + n.Entries = append(n.Entries, newEntry) + } else { + n.Entries = slices.Insert(n.Entries, idx, newEntry) + } + return n, nil, nil + } + + // we need to split + e := n.Entries[idx] + left, right, err := e.Child.split(key) + if err != nil { + return nil, nil, err + } + // remove the existing entry, and replace with three new entries + n.Entries = slices.Delete(n.Entries, idx, idx+1) + n.Entries = slices.Insert( + n.Entries, + idx, + NodeEntry{Child: left, Dirty: true}, + newEntry, + NodeEntry{Child: right, Dirty: true}, + ) + return n, nil, nil +} + +func (n *Node) splitEntries(idx int) (*Node, *Node, error) { + if idx == 0 || idx >= len(n.Entries) { + return nil, nil, fmt.Errorf("splitting at one end or the other of entries") + } + left := Node{ + Height: n.Height, + Dirty: true, + Entries: n.Entries[:idx], + } + right := Node{ + Height: n.Height, + Dirty: true, + // don't use the same slice here + Entries: append([]NodeEntry{}, n.Entries[idx:]...), + } + if left.IsEmpty() || right.IsEmpty() { + return nil, nil, fmt.Errorf("one of the legs is empty (idx=%d, len=%d)", idx, len(n.Entries)) + } + return &left, &right, nil +} + +func (n *Node) split(key []byte) (*Node, *Node, error) { + if n.IsEmpty() { + // TODO: this feels defensive and could be removed + return nil, nil, fmt.Errorf("tried to split an empty node") + } + + idx, split, err := n.findInsertionIndex(key) + if err != nil { + return nil, nil, err + } + if !split { + // simple split based on values + return n.splitEntries(idx) + } + + // need to split recursively + e := n.Entries[idx] + lowerLeft, lowerRight, err := e.Child.split(key) + if err != nil { + return nil, nil, err + } + left := &Node{ + Height: n.Height, + Dirty: true, + Entries: []NodeEntry{}, + } + left.Entries = append(left.Entries, n.Entries[:idx]...) + left.Entries = append(left.Entries, NodeEntry{Child: lowerLeft, Dirty: true}) + right := &Node{ + Height: n.Height, + Dirty: true, + Entries: []NodeEntry{NodeEntry{Child: lowerRight, Dirty: true}}, + } + if idx+1 < len(n.Entries) { + right.Entries = append(right.Entries, n.Entries[idx+1:]...) + } + return left, right, nil +} + +// inserts a node "above" this node in tree, possibly splitting the current node +func (n *Node) insertParent(key []byte, val cid.Cid, height int) (*Node, *cid.Cid, error) { + var parent *Node + if n.IsEmpty() { + // if current node is empty, just replace directly with current height + parent = &Node{ + Height: height, + Dirty: true, + } + } else { + // otherwise push a layer and recurse + parent = &Node{ + Height: n.Height + 1, + Dirty: true, + Entries: []NodeEntry{NodeEntry{ + Child: n, + Dirty: true, + }}, + } + } + // regular insertion will handle any necessary "split" + return parent.insert(key, val, height) +} + +// inserts a node "below" this node in tree; either creating a new child entry or re-using an existing one +func (n *Node) insertChild(key []byte, val cid.Cid, height int) (*Node, *cid.Cid, error) { + // look for an existing child node which encompasses the key, and use that + idx := n.findExistingChild(key) + if idx >= 0 { + e := n.Entries[idx] + if e.Child == nil { + return nil, nil, fmt.Errorf("could not insert key: %w", ErrPartialTree) + } + newChild, prev, err := e.Child.insert(key, val, height) + if err != nil { + return nil, nil, err + } + if prev != nil && *prev == val { + // no-op + return n, &val, nil + } + n.Dirty = true + n.Entries[idx].Child = newChild + n.Entries[idx].Dirty = true + return n, prev, nil + } + + // insert a new child node. this might be recursive if the child is not a *direct* child + idx, split, err := n.findInsertionIndex(key) + if err != nil { + return nil, nil, err + } + if split { + return nil, nil, fmt.Errorf("unexpected split when inserting child") + } + n.Dirty = true + newChild := &Node{ + Height: n.Height - 1, + Dirty: true, + } + newChild, _, err = newChild.insert(key, val, height) + if err != nil { + return nil, nil, err + } + newEntry := NodeEntry{ + Child: newChild, + Dirty: true, + } + if idx == len(n.Entries) { + n.Entries = append(n.Entries, newEntry) + } else { + n.Entries = slices.Insert(n.Entries, idx, newEntry) + } + return n, nil, nil +} diff --git a/atproto/repo/mst/node_remove.go b/atproto/repo/mst/node_remove.go new file mode 100644 index 000000000..7741bdac3 --- /dev/null +++ b/atproto/repo/mst/node_remove.go @@ -0,0 +1,153 @@ +package mst + +import ( + "errors" + "fmt" + "slices" + + "github.com/ipfs/go-cid" +) + +// Removes key/value from the sub-tree provided, returning a new tree, and the previous CID value. If key is not found, returns unmodified subtree, and nil for the returned CID. +// +// n: Node at top of sub-tree to operate on. Must not be nil. +// key: key or path being inserted. must not be empty/nil +// height: tree height corresponding to key. if a negative value is provided, will be computed; use -1 instead of 0 if height is not known +func (n *Node) remove(key []byte, height int) (*Node, *cid.Cid, error) { + if n.Stub { + return nil, nil, ErrPartialTree + } + // TODO: do we need better handling of "is this the top"? + top := false + if height < 0 { + top = true + height = HeightForKey(key) + } + + if height > n.Height { + // removing a key from a higher layer; key was not in tree + return n, nil, nil + } + + if height < n.Height { + // TODO: handle case of this returning an empty node at top of tree, with wrong height + return n.removeChild(key, height) + } + + // look at this level + idx := n.findExistingEntry(key) + if idx < 0 { + // key not found + return n, nil, nil + } + + // found it! will remove from list + n.Dirty = true + prev := n.Entries[idx].Value + + // check if we need to "merge" adjacent nodes + if idx > 0 && idx+1 < len(n.Entries) && n.Entries[idx-1].IsChild() && n.Entries[idx+1].IsChild() { + if n.Entries[idx-1].Child == nil || n.Entries[idx+1].Child == nil { + return nil, nil, fmt.Errorf("can not merge child nodes: %w", ErrPartialTree) + } + newChild, err := mergeNodes(n.Entries[idx-1].Child, n.Entries[idx+1].Child) + if err != nil { + return nil, nil, err + } + n.Entries = slices.Delete(n.Entries, idx, idx+2) + n.Entries[idx-1] = NodeEntry{Child: newChild, Dirty: true} + } else { + // simple removal + n.Entries = slices.Delete(n.Entries, idx, idx+1) + } + + // marks adjacent child nodes dirty to include as "proof" + if err := proveMutation(n, key); err != nil && !errors.Is(err, ErrPartialTree) { + return nil, nil, err + } + + // check if top of node is now just a pointer + if top { + for { + if len(n.Entries) != 1 || !n.Entries[0].IsChild() { + break + } + if n.Entries[0].Child == nil { + // this is something of a hack, for MST inversion which requires trimming the tree + if n.Entries[0].ChildCID == nil { + return nil, nil, fmt.Errorf("can not prune top of tree: %w", ErrPartialTree) + } else { + n = &Node{ + Height: n.Height - 1, + Stub: true, + CID: n.Entries[0].ChildCID, + } + } + } else { + n = n.Entries[0].Child + } + } + } + return n, prev, nil +} + +func mergeNodes(left *Node, right *Node) (*Node, error) { + idx := len(left.Entries) + n := &Node{ + Height: left.Height, + Dirty: true, + Entries: append(left.Entries, right.Entries...), + } + if n.Entries[idx-1].IsChild() && n.Entries[idx].IsChild() { + // need to merge recursively + lowerLeft := n.Entries[idx-1] + lowerRight := n.Entries[idx] + if lowerLeft.Child == nil || lowerRight.Child == nil { + return nil, fmt.Errorf("can not merge child nodes: %w", ErrPartialTree) + } + lowerMerged, err := mergeNodes(lowerLeft.Child, lowerRight.Child) + if err != nil { + return nil, err + } + n.Entries[idx-1] = NodeEntry{Child: lowerMerged, Dirty: true} + n.Entries = slices.Delete(n.Entries, idx, idx+1) + } + return n, nil +} + +// internal helper +func (n *Node) removeChild(key []byte, height int) (*Node, *cid.Cid, error) { + // look for a child + idx := n.findExistingChild(key) + if idx < 0 { + // no child pointer; key not in tree + return n, nil, nil + } + + e := n.Entries[idx] + if e.Child == nil { + // partial node, can't recurse + return nil, nil, fmt.Errorf("could not remove key: %w", ErrPartialTree) + } + newChild, prev, err := e.Child.remove(key, height) + if err != nil { + return nil, nil, err + } + if prev == nil { + // no-op + return n, nil, nil + } + + // if the child node was updated, but still exists, just update pointer + if !newChild.IsEmpty() { + n.Dirty = true + n.Entries[idx].Child = newChild + n.Entries[idx].Dirty = true + return n, prev, nil + } + + // if new child was empty, remove it from entry list; note that *this* entry might now be empty + n.Dirty = true + n.Entries = slices.Delete(n.Entries, idx, idx+1) + return n, prev, nil +} diff --git a/atproto/repo/mst/testdata/example_keys.txt b/atproto/repo/mst/testdata/example_keys.txt new file mode 100644 index 000000000..b34212ac0 --- /dev/null +++ b/atproto/repo/mst/testdata/example_keys.txt @@ -0,0 +1,156 @@ +A0/374913 +A1/076595 +A2/827942 +A3/578971 +A4/055903 +A5/518415 +B0/601692 +B1/986427 +B2/827649 +B3/095483 +B4/774183 +B5/116729 +C0/451630 +C1/438573 +C2/014073 +C3/564755 +C4/134079 +C5/141153 +D0/952776 +D1/834852 +D2/269196 +D3/038750 +D4/052059 +D5/563177 +E0/670489 +E1/091396 +E2/819540 +E3/391311 +E4/820614 +E5/512478 +F0/697858 +F1/085263 +F2/483591 +F3/409933 +F4/789697 +F5/271416 +G0/765327 +G1/209912 +G2/611528 +G3/649394 +G4/585887 +G5/298495 +H0/131238 +H1/566929 +H2/618272 +H3/500151 +H4/841548 +H5/642354 +I0/536928 +I1/525517 +I2/800680 +I3/818503 +I4/561177 +I5/010047 +J0/453243 +J1/217783 +J2/960389 +J3/501274 +J4/042054 +J5/743154 +K0/125271 +K1/317361 +K2/453868 +K3/214010 +K4/164720 +K5/177856 +L0/502889 +L1/574576 +L2/596333 +L3/683657 +L4/724989 +L5/093883 +M0/141744 +M1/643368 +M2/919782 +M3/836327 +M4/177463 +M5/563354 +N0/370604 +N1/563732 +N2/177587 +N3/678428 +N4/599183 +N5/567564 +O0/523870 +O1/052141 +O2/037651 +O3/773808 +O4/140952 +O5/318605 +P0/133157 +P1/394633 +P2/521462 +P3/493488 +P4/908754 +P5/109455 +Q0/835234 +Q1/131542 +Q2/680035 +Q3/253381 +Q4/019053 +Q5/658167 +R0/129386 +R1/363149 +R2/742766 +R3/039235 +R4/482275 +R5/817312 +S0/340283 +S1/561525 +S2/914574 +S3/909434 +S4/789708 +S5/803866 +T0/255204 +T1/716687 +T2/256231 +T3/054247 +T4/419247 +T5/509584 +U0/298296 +U1/851680 +U2/342856 +U3/597327 +U4/311686 +U5/030156 +V0/221100 +V1/741554 +V2/267990 +V3/674163 +V4/739931 +V5/573718 +W0/034202 +W1/697411 +W2/460313 +W3/189647 +W4/847299 +W5/648086 +X0/287498 +X1/044093 +X2/613770 +X3/577587 +X4/779391 +X5/339246 +Y0/986350 +Y1/044567 +Y2/478044 +Y3/757097 +Y4/396913 +Y5/802264 +Z0/425878 +Z1/127557 +Z2/441927 +Z3/064474 +Z4/888344 +Z5/977983 diff --git a/atproto/repo/mst/tree.go b/atproto/repo/mst/tree.go new file mode 100644 index 000000000..bb3020582 --- /dev/null +++ b/atproto/repo/mst/tree.go @@ -0,0 +1,171 @@ +package mst + +import ( + "context" + "errors" + "fmt" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" +) + +// High-level API for an MST, as a decoded in-memory data structure. +// +// This might be an entire tree (all child nodes in-memory), or might be a partial tree with some nodes as CID links. Operations on the tree do not persist to any backing storage automatically. +// +// Errors when operating on the tree may leave the tree in a partially modified or invalid/corrupt state. +type Tree struct { + Root *Node + // TODO: have a blockstore.Blockstore for loading lazily? +} + +var ErrInvalidKey = errors.New("bytestring not a valid MST key") + +var ErrPartialTree = errors.New("MST is not complete") + +var ErrInvalidTree = errors.New("invalid MST structure") + +func NewEmptyTree() Tree { + return Tree{ + Root: &Node{ + Dirty: true, + Height: 0, + }, + } +} + +// Adds a key/value to the tree, and returns any previously existing value (CID). +// +// Caller can inspect the previous value to determine if the behavior was a "creation" (key didn't exist), an "update" (key existed with a different value), or no-op (key existed with current value). +// +// key: key or path being inserted. must not be empty/nil +// val: CID value being inserted +func (t *Tree) Insert(key []byte, val cid.Cid) (*cid.Cid, error) { + if !IsValidKey(key) { + return nil, ErrInvalidKey + } + out, prev, err := t.Root.insert(key, val, -1) + if err != nil { + return nil, err + } + t.Root = out + return prev, nil +} + +// Removes key/value from the sub-tree provided. Return the previous CID value, if any. If key was not found, returns nil (which is not an error). +// +// key: key or path being inserted. must not be empty/nil +func (t *Tree) Remove(key []byte) (*cid.Cid, error) { + if !IsValidKey(key) { + return nil, ErrInvalidKey + } + out, prev, err := t.Root.remove(key, -1) + if err != nil { + return nil, err + } + t.Root = out + return prev, nil +} + +// Reads the value (CID) corresponding to the key. +// +// If key is not in the tree, returns nil, not an error. +// +// key: key or path being inserted. must not be empty/nil +func (t *Tree) Get(key []byte) (*cid.Cid, error) { + if !IsValidKey(key) { + return nil, ErrInvalidKey + } + return t.Root.getCID(key, -1) +} + +// Walks the Tree, invoking the callback function on each key/value pair. +func (t *Tree) Walk(f func(key []byte, val cid.Cid) error) error { + return t.Root.walk(f) +} + +// Creates a new Tree by loading key/value pairs from a map. +func LoadTreeFromMap(m map[string]cid.Cid) (*Tree, error) { + if m == nil { + return nil, fmt.Errorf("un-initialized map as an argument") + } + t := NewEmptyTree() + var err error + for key, val := range m { + _, err = t.Insert([]byte(key), val) + if err != nil { + return nil, fmt.Errorf("unexpected failure to build MST structure: %w", err) + } + } + return &t, nil +} + +// Recursively walks the tree and writes key/value pairs to map `m` +// +// The map (`m`) is mutated in place (by reference); the map must be initialized before calling. +func (t *Tree) WriteToMap(m map[string]cid.Cid) error { + if m == nil { + return fmt.Errorf("un-initialized map as an argument") + } + if t.Root == nil { + return fmt.Errorf("empty tree root") + } + return t.Root.writeToMap(m) +} + +// Returns the overall root-node CID for the MST. +// +// If possible, lazily returned a known value. If necessary, recursively encodes tree nodes to compute CIDs. +// +// NOTE: will mark the tree "clean" (clear any dirty flags). +func (t *Tree) RootCID() (*cid.Cid, error) { + if t.Root != nil && t.Root.Stub && !t.Root.Dirty && t.Root.CID != nil { + return t.Root.CID, nil + } + return t.Root.writeBlocks(context.Background(), nil, true) +} + +// If the tree contains no key/value pairs, returns true. +func (t *Tree) IsEmpty() bool { + if t.Root == nil { + return true + } + return t.Root.IsEmpty() +} + +// Returns false if all nodes in the tree are available in-memory in decoded format; otherwise returns true. Does not consider record data, only MST nodes. +func (t *Tree) IsPartial() bool { + if t.Root == nil { + return true + } + return t.Root.IsPartial() +} + +// Creates a deep copy of MST +func (t *Tree) Copy() Tree { + return Tree{ + Root: t.Root.deepCopy(), + } +} + +func LoadTreeFromStore(ctx context.Context, bs MSTBlockSource, root cid.Cid) (*Tree, error) { + n, err := loadNodeFromStore(ctx, bs, root) + if err != nil { + return nil, err + } + n.ensureHeights() + return &Tree{ + Root: n, + }, nil +} + +// subset of Blockstore that we actually need +type MSTBlockSource interface { + Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) +} + +// Walks the tree, encodes any "dirty" nodes as CBOR data, and writes that data as blocks to the provided blockstore. Returns root CID. +func (t *Tree) WriteDiffBlocks(ctx context.Context, bs blockstore.Blockstore) (*cid.Cid, error) { + return t.Root.writeBlocks(ctx, bs, true) +} diff --git a/atproto/repo/mst/util.go b/atproto/repo/mst/util.go new file mode 100644 index 000000000..73bbad6b0 --- /dev/null +++ b/atproto/repo/mst/util.go @@ -0,0 +1,57 @@ +package mst + +import ( + "crypto/sha256" +) + +const ( + // Maximum length, in bytes, of a key in the tree. Note that the atproto specifications imply a repo path maximum length, but don't say anything directly about MST key, other than they can not be empty (zero-length). + MAX_KEY_BYTES = 1024 +) + +// Computes the MST "height" for a key (bytestring). Layers are counted from the "bottom" of the tree, starting with zero. +// +// For atproto repository v3, uses SHA-256 as the hashing function and counts two bits at a time, for an MST "fanout" value of 16. +func HeightForKey(key []byte) (height int) { + hv := sha256.Sum256(key) + for _, b := range hv { + if b&0xC0 != 0 { + // Common case. No leading pair of zero bits. + break + } + if b == 0x00 { + height += 4 + continue + } + if b&0xFC == 0x00 { + height += 3 + } else if b&0xF0 == 0x00 { + height += 2 + } else { + height += 1 + } + break + } + return height +} + +// Computes the common prefix length between two bytestrings. +// +// Used when compacting node entry lists for encoding. +func CountPrefixLen(a, b []byte) int { + // This pattern avoids panicindex calls, as the Go compiler's prove pass can convince itself that neither a[i] nor b[i] are ever out of bounds. + var i int + for i = 0; i < len(a) && i < len(b); i++ { + if a[i] != b[i] { + return i + } + } + return i +} + +func IsValidKey(key []byte) bool { + if len(key) == 0 || len(key) > MAX_KEY_BYTES { + return false + } + return true +} diff --git a/atproto/repo/mst/util_interop_test.go b/atproto/repo/mst/util_interop_test.go new file mode 100644 index 000000000..73362a3ad --- /dev/null +++ b/atproto/repo/mst/util_interop_test.go @@ -0,0 +1,100 @@ +package mst + +import ( + "bufio" + "os" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPrefixLen(t *testing.T) { + msg := "length of common prefix between strings" + + testVec := []struct { + Left []byte + Right []byte + Len int + }{ + {[]byte(""), []byte(""), 0}, + {[]byte("abc"), []byte("abc"), 3}, + {[]byte(""), []byte("abc"), 0}, + {[]byte("abc"), []byte(""), 0}, + {[]byte("ab"), []byte("abc"), 2}, + {[]byte("abc"), []byte("ab"), 2}, + {[]byte("abcde"), []byte("abc"), 3}, + {[]byte("abc"), []byte("abcde"), 3}, + {[]byte("abcde"), []byte("abc1"), 3}, + {[]byte("abcde"), []byte("abb"), 2}, + {[]byte("abcde"), []byte("qbb"), 0}, + {[]byte("abc"), []byte("abc\x00"), 3}, + {[]byte("abc\x00"), []byte("abc"), 3}, + } + + for _, c := range testVec { + assert.Equal(t, c.Len, CountPrefixLen(c.Left, c.Right), msg) + } +} + +func TestPrefixLenWide(t *testing.T) { + // NOTE: these are not cross-language consistent! + msg := "length of common prefix between strings (wide chars)" + + assert.Equal(t, 9, len("jalapeño"), msg) // 8 in javascript + assert.Equal(t, 4, len("💩"), msg) // 2 in javascript + assert.Equal(t, 18, len("👩‍👧‍👧"), msg) // 8 in javascript + + testVec := []struct { + Left []byte + Right []byte + Len int + }{ + {[]byte(""), []byte(""), 0}, + {[]byte("jalapeño"), []byte("jalapeno"), 6}, + {[]byte("jalapeñoA"), []byte("jalapeñoB"), 9}, + {[]byte("coöperative"), []byte("coüperative"), 3}, + {[]byte("abc💩abc"), []byte("abcabc"), 3}, + {[]byte("💩abc"), []byte("💩ab"), 6}, + {[]byte("abc👩‍👦‍👦de"), []byte("abc👩‍👧‍👧de"), 13}, + } + + for _, c := range testVec { + assert.Equal(t, c.Len, CountPrefixLen(c.Left, c.Right), msg) + } +} + +func TestHeightForKey(t *testing.T) { + assert := assert.New(t) + msg := "MST 'depth' computation (SHA-256 leading zeros)" + assert.Equal(HeightForKey([]byte("")), 0, msg) + assert.Equal(HeightForKey([]byte("asdf")), 0, msg) + assert.Equal(HeightForKey([]byte("blue")), 1, msg) + assert.Equal(HeightForKey([]byte("2653ae71")), 0, msg) + assert.Equal(HeightForKey([]byte("88bfafc7")), 2, msg) + assert.Equal(HeightForKey([]byte("2a92d355")), 4, msg) + assert.Equal(HeightForKey([]byte("884976f5")), 6, msg) + assert.Equal(HeightForKey([]byte("app.bsky.feed.post/454397e440ec")), 4, msg) + assert.Equal(HeightForKey([]byte("app.bsky.feed.post/9adeb165882c")), 8, msg) + + assert.Equal(HeightForKey([]byte("R2/359107")), 2, msg) +} + +func TestExampleKeyHeights(t *testing.T) { + assert := assert.New(t) + + file, err := os.Open("testdata/example_keys.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) < 3 || line[0] == '#' { + continue + } + height, err := strconv.Atoi(string(line[1])) + assert.NoError(err) + assert.Equal(height, HeightForKey([]byte(line))) + } + assert.NoError(scanner.Err()) +} diff --git a/atproto/repo/mst/verify.go b/atproto/repo/mst/verify.go new file mode 100644 index 000000000..25ec0030b --- /dev/null +++ b/atproto/repo/mst/verify.go @@ -0,0 +1,84 @@ +package mst + +import ( + "bytes" + "fmt" +) + +func (t *Tree) Verify() error { + if t.Root == nil { + return fmt.Errorf("tree missing root node") + } + return t.Root.verifyStructure(-1, nil) +} + +func (n *Node) verifyStructure(height int, key []byte) error { + if n == nil { + return fmt.Errorf("nil node") + } + if n.Stub { + return fmt.Errorf("stub node") + } + if n.CID == nil && n.Dirty == false { + return fmt.Errorf("node missing CID, but not marked dirty") + } + if len(n.Entries) == 0 { + if height >= 0 { + return fmt.Errorf("empty tree node") + } + // entire tree is empty + return nil + } + + if height < 0 { + // do a quick pass to compute current height + for _, e := range n.Entries { + if e.IsValue() { + height = HeightForKey(e.Key) + break + } + } + } + if height < 0 { + return fmt.Errorf("top of tree is just a pointer to child") + } + if n.Height == -1 || n.Height != height { + return fmt.Errorf("node has incorrect height: %d", n.Height) + } + + lastWasChild := false + for _, e := range n.Entries { + if e.IsChild() { + if lastWasChild { + return fmt.Errorf("sibling children in entries list") + } + lastWasChild = true + if e.IsValue() { + return fmt.Errorf("entry is both a child and a value") + } + if height == 0 { + return fmt.Errorf("child below zero height") + } + if e.Child != nil { + if err := e.Child.verifyStructure(height-1, key); err != nil { + return err + } + } + } else if e.IsValue() { + lastWasChild = false + if bytes.Equal(key, e.Key) { + return fmt.Errorf("duplicate key in tree") + } + if bytes.Compare(key, e.Key) > 0 { + return fmt.Errorf("out of order keys") + } + key = e.Key + if height != HeightForKey(e.Key) { + return fmt.Errorf("wrong height for key: %d", HeightForKey(e.Key)) + } + } else { + return fmt.Errorf("entry was neither child nor value") + } + } + return nil +} diff --git a/atproto/repo/operation.go b/atproto/repo/operation.go new file mode 100644 index 000000000..37476c48a --- /dev/null +++ b/atproto/repo/operation.go @@ -0,0 +1,158 @@ +package repo + +import ( + "fmt" + "sort" + + "github.com/bluesky-social/indigo/atproto/repo/mst" + + "github.com/ipfs/go-cid" +) + +// Metadata about update to a single record (key) in the repo. +// +// Used as an abstraction for creating or validating "commit diffs" (eg, `#commit` firehose events) +type Operation struct { + // key of the record, eg, '{collection}/{record-key}' + Path string + // the new record CID value (or nil if this is a deletion) + Value *cid.Cid + // the previous record CID value (or nil if this is a creation) + Prev *cid.Cid +} + +func (op *Operation) IsCreate() bool { + if op.Value != nil && op.Prev == nil { + return true + } + return false +} + +func (op *Operation) IsUpdate() bool { + if op.Value != nil && op.Prev != nil && *op.Value != *op.Prev { + return true + } + return false +} + +func (op *Operation) IsDelete() bool { + if op.Value == nil && op.Prev != nil { + return true + } + return false +} + +// Mutates the tree, returning a full `Operation` +func ApplyOp(tree *mst.Tree, path string, val *cid.Cid) (*Operation, error) { + if val != nil { + prev, err := tree.Insert([]byte(path), *val) + if err != nil { + return nil, err + } + op := &Operation{ + Path: path, + Value: val, + Prev: prev, + } + return op, nil + } else { + prev, err := tree.Remove([]byte(path)) + if err != nil { + return nil, err + } + op := &Operation{ + Path: path, + Value: val, + Prev: prev, + } + return op, nil + } +} + +// Does a simple "forwards" (not inversion) check of operation +func CheckOp(tree *mst.Tree, op *Operation) error { + val, err := tree.Get([]byte(op.Path)) + if err != nil { + return err + } + if op.IsCreate() || op.IsUpdate() { + if val == nil || *val != *op.Value { + return fmt.Errorf("tree value did not match op: %s %s", op.Path, val) + } + return nil + } + if op.IsDelete() { + if val != nil { + return fmt.Errorf("key still in tree after deletion op: %s", op.Path) + } + return nil + } + return fmt.Errorf("invalid operation") +} + +// Applies the inversion of the `op` to the `tree`. This mutates the tree. +func InvertOp(tree *mst.Tree, op *Operation) error { + if op.IsCreate() { + prev, err := tree.Remove([]byte(op.Path)) + if err != nil { + return fmt.Errorf("failed to invert op: %w", err) + } + if prev == nil || *prev != *op.Value { + return fmt.Errorf("failed to invert creation: previous record CID didn't match") + } + return nil + } + if op.IsUpdate() { + prev, err := tree.Insert([]byte(op.Path), *op.Prev) + if err != nil { + return fmt.Errorf("failed to invert op: %w", err) + } + if prev == nil || *prev != *op.Value { + return fmt.Errorf("failed to invert update: previous record CID didn't match") + } + return nil + } + if op.IsDelete() { + prev, err := tree.Insert([]byte(op.Path), *op.Prev) + if err != nil { + return fmt.Errorf("failed to invert op: %w", err) + } + if prev != nil { + return fmt.Errorf("failed to invert deletion: key was previously in tree") + } + return nil + } + return fmt.Errorf("invalid operation") +} + +type opByPath []Operation + +func (a opByPath) Len() int { return len(a) } +func (a opByPath) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +func (a opByPath) Less(i, j int) bool { + + // sort deletions first + if a[i].IsDelete() && !a[j].IsDelete() { + return true + } + + // then by path + return a[i].Path < a[j].Path +} + +// re-orders operation list, and checks for duplicates +func NormalizeOps(list []Operation) ([]Operation, error) { + // TODO: can this just use the slice ref, instead of returning? + + set := map[string]bool{} + for _, op := range list { + if _, ok := set[op.Path]; ok { + return nil, fmt.Errorf("duplicate path in operation list") + } + set[op.Path] = true + } + + sort.Sort(opByPath(list)) + return list, nil +} diff --git a/atproto/repo/operation_test.go b/atproto/repo/operation_test.go new file mode 100644 index 000000000..48fc483ea --- /dev/null +++ b/atproto/repo/operation_test.go @@ -0,0 +1,295 @@ +package repo + +import ( + "context" + "encoding/hex" + "fmt" + "math/rand" + "testing" + + "github.com/bluesky-social/indigo/atproto/repo/mst" + + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" +) + +func randomCid() cid.Cid { + buf := make([]byte, 32) + rand.Read(buf) + c, err := cid.NewPrefixV1(cid.DagCBOR, multihash.SHA2_256).Sum(buf) + if err != nil { + panic(err) + } + return c +} + +func randomStr() string { + buf := make([]byte, 16) + rand.Read(buf) + return hex.EncodeToString(buf) +} + +func debugCountEntries(n *mst.Node) int { + if n == nil { + return 0 + } + count := 0 + for _, e := range n.Entries { + if e.IsValue() { + count++ + } + if e.IsChild() && e.Child != nil { + count += debugCountEntries(e.Child) + } + } + return count +} + +func TestBasicOperation(t *testing.T) { + assert := assert.New(t) + + c2, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222") + c3, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu333333333") + et := mst.NewEmptyTree() + tree := &et + var op *Operation + var err error + + op, err = ApplyOp(tree, "color/green", &c2) + assert.NoError(err) + assert.True(op.IsCreate()) + assert.NoError(CheckOp(tree, op)) + + op, err = ApplyOp(tree, "color/brown", &c2) + assert.NoError(err) + assert.True(op.IsCreate()) + assert.NoError(CheckOp(tree, op)) + + op, err = ApplyOp(tree, "color/brown", &c3) + assert.NoError(err) + assert.True(op.IsUpdate()) + assert.Equal(c3, *op.Value) + assert.Equal(c2, *op.Prev) + assert.NoError(CheckOp(tree, op)) + + op, err = ApplyOp(tree, "color/brown", nil) + assert.NoError(err) + assert.True(op.IsDelete()) + assert.NoError(CheckOp(tree, op)) + err = InvertOp(tree, op) + assert.NoError(err) + assert.Error(CheckOp(tree, op)) + + op, err = ApplyOp(tree, "color/orange", &c3) + assert.NoError(err) + assert.True(op.IsCreate()) + assert.NoError(CheckOp(tree, op)) + err = InvertOp(tree, op) + assert.NoError(err) + assert.Error(CheckOp(tree, op)) + + _, err = ApplyOp(tree, "color/pink", &c3) + assert.NoError(err) + op, err = ApplyOp(tree, "color/pink", &c2) + assert.NoError(err) + assert.NoError(CheckOp(tree, op)) + assert.True(op.IsUpdate()) + err = InvertOp(tree, op) + assert.NoError(err) + assert.Error(CheckOp(tree, op)) +} + +func TestRandomOperations(t *testing.T) { + // single-op commits, near-empty repo + randomOperations(t, 1, 1, 50) + // single-op commits, large repo + randomOperations(t, 10000, 1, 50) + // multi-op commit + randomOperations(t, 2000, 8, 50) +} + +func randomOperations(t *testing.T, size, opCount, iterations int) { + assert := assert.New(t) + ctx := context.Background() + + // generate a random starting tree + startMap := make(map[string]cid.Cid, size) + for range size { + k := randomStr() + // ensure key is not already in the random set + for { + _, ok := startMap[k] + if !ok { + break + } + k = randomStr() + } + startMap[k] = randomCid() + } + mapKeys := make([]string, len(startMap)) + i := 0 + for k := range startMap { + mapKeys[i] = k + i++ + } + rand.Shuffle(len(mapKeys), func(i, j int) { + mapKeys[i], mapKeys[j] = mapKeys[j], mapKeys[i] + }) + tree, err := mst.LoadTreeFromMap(startMap) + if err != nil { + t.Fatal(err) + } + assert.Equal(size, debugCountEntries(tree.Root)) + assert.NoError(tree.Verify()) + + for range iterations { + // compute CID of the tree + startCID, err := tree.RootCID() + if err != nil { + t.Fatal(err) + } + + // do some random ops + opSet := []Operation{} + var op *Operation + c := randomCid() + for range opCount { + // creations + op, err = ApplyOp(tree, randomStr(), &c) + assert.NoError(err) + opSet = append(opSet, *op) + } + + for range opCount { + // deletions + op, err = ApplyOp(tree, mapKeys[rand.Intn(len(mapKeys))], nil) + assert.NoError(err) + if op.Prev != nil { + opSet = append(opSet, *op) + } + } + + for range opCount { + // updates (must happen after deletions!) + k := mapKeys[rand.Intn(len(mapKeys))] + v, err := tree.Get([]byte(k)) + assert.NoError(err) + if v != nil && *v != c { + op, err = ApplyOp(tree, k, &c) + assert.NoError(err) + assert.Equal(*v, *op.Prev) + assert.Equal(c, *op.Value) + if op.Prev != nil { + opSet = append(opSet, *op) + } + } + } + + // extract diff as separate tree, and validate that + diffBlocks := blockstore.NewBlockstore(datastore.NewMapDatastore()) + diffRoot, err := tree.WriteDiffBlocks(ctx, diffBlocks) + if err != nil { + t.Fatal(err) + } + diffTree, err := mst.LoadTreeFromStore(ctx, diffBlocks, *diffRoot) + if err != nil { + t.Fatal(err) + } + assert.NoError(tree.Verify()) + + // re-compute partial commit (not related to main test path) + diffCID, err := tree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(*diffRoot, *diffCID) + + // uncomment this to try inverting on full tree + //diffTree = tree + + // check all ops against full tree (this is a redundant check) + for _, op := range opSet { + assert.NoError(CheckOp(tree, &op)) + } + + // sort ops (comment to disable) + opSet, err = NormalizeOps(opSet) + if err != nil { + t.Fatal(err) + } + + // invert operations + for i, op := range opSet { + err := CheckOp(diffTree, &op) + fmt.Printf("loop=%d key=%s val=%s prev=%s\n", i, op.Path, op.Value, op.Prev) + assert.NoError(diffTree.Verify()) + if err != nil { + //debugPrintTree(diffTree, 0) + t.Fatal(err) + } + + err = InvertOp(diffTree, &op) + assert.NoError(err) + if err != nil { + t.Fatal(err) + } + } + + finalCID, err := diffTree.RootCID() + if err != nil { + t.Fatal(err) + } + assert.Equal(*startCID, *finalCID) + } + + // fiddle this to purge test cache + _ = 12 +} + +func TestNormalizeOps(t *testing.T) { + assert := assert.New(t) + + c2, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222") + c3, _ := cid.Decode("bafyreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu333333333") + + simple := []Operation{ + Operation{ + Path: "create-BBB", + Value: &c2, + Prev: nil, + }, + Operation{ + Path: "create-AAA", + Value: &c2, + Prev: nil, + }, + Operation{ + Path: "delete-me", + Value: nil, + Prev: &c2, + }, + } + out, err := NormalizeOps(simple) + assert.NoError(err) + assert.Equal(3, len(out)) + assert.Equal("delete-me", out[0].Path) + assert.Equal("create-BBB", out[2].Path) + + dupes := []Operation{ + Operation{ + Path: "create-BBB", + Value: nil, + Prev: &c2, + }, + Operation{ + Path: "create-BBB", + Value: nil, + Prev: &c3, + }, + } + _, err = NormalizeOps(dupes) + assert.Error(err) +} diff --git a/atproto/repo/repo.go b/atproto/repo/repo.go new file mode 100644 index 000000000..0273e044c --- /dev/null +++ b/atproto/repo/repo.go @@ -0,0 +1,82 @@ +package repo + +import ( + "context" + "errors" + + "github.com/bluesky-social/indigo/atproto/repo/mst" + "github.com/bluesky-social/indigo/atproto/syntax" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" +) + +// Version of the repo data format implemented in this package +const ATPROTO_REPO_VERSION int64 = 3 + +// High-level wrapper struct for an atproto repository. +type Repo struct { + DID syntax.DID + Clock *syntax.TIDClock + + RecordStore RepoBlockSource // formerly blockstore.Blockstore + MST mst.Tree +} + +// subset of Blockstore that we actually need +type RepoBlockSource interface { + Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) +} + +var ErrNotFound = errors.New("record not found in repository") + +//func NewEmptyRepo(did syntax.DID) Repo { +// clk := syntax.NewTIDClock(0) +// return Repo{ +// DID: did, +// Clock: &clk, +// RecordStore: blockstore.NewBlockstore(datastore.NewMapDatastore()), +// MST: mst.NewEmptyTree(), +// } +//} + +func (repo *Repo) GetRecordCID(ctx context.Context, collection syntax.NSID, rkey syntax.RecordKey) (*cid.Cid, error) { + path := collection.String() + "/" + rkey.String() + c, err := repo.MST.Get([]byte(path)) + if err != nil { + return nil, err + } + if c == nil { + return nil, ErrNotFound + } + return c, nil +} + +func (repo *Repo) GetRecordBytes(ctx context.Context, collection syntax.NSID, rkey syntax.RecordKey) ([]byte, *cid.Cid, error) { + c, err := repo.GetRecordCID(ctx, collection, rkey) + if err != nil { + return nil, nil, err + } + blk, err := repo.RecordStore.Get(ctx, *c) + if err != nil { + return nil, nil, err + } + // TODO: not verifying CID + return blk.RawData(), c, nil +} + +// Snapshots the current state of the repository, resulting in a new (unsigned) `Commit` struct. +func (repo *Repo) Commit() (*Commit, error) { + root, err := repo.MST.RootCID() + if err != nil { + return nil, err + } + c := Commit{ + DID: repo.DID.String(), + Version: ATPROTO_REPO_VERSION, + Prev: nil, + Data: *root, + Rev: repo.Clock.Next().String(), + } + return &c, nil +} diff --git a/atproto/repo/sync.go b/atproto/repo/sync.go new file mode 100644 index 000000000..460d737d2 --- /dev/null +++ b/atproto/repo/sync.go @@ -0,0 +1,208 @@ +package repo + +import ( + "bytes" + "context" + "fmt" + "log/slog" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/ipfs/go-cid" +) + +// temporary/experimental method to parse and verify a firehose commit message. +// +// TODO: move to a separate 'sync' package? break up in to smaller components? +func VerifyCommitMessage(ctx context.Context, msg *comatproto.SyncSubscribeRepos_Commit) (*Repo, error) { + + logger := slog.Default().With("did", msg.Repo, "rev", msg.Rev, "seq", msg.Seq, "time", msg.Time) + + did, err := syntax.ParseDID(msg.Repo) + if err != nil { + return nil, err + } + rev, err := syntax.ParseTID(msg.Rev) + if err != nil { + return nil, err + } + _, err = syntax.ParseDatetime(msg.Time) + if err != nil { + return nil, err + } + + if msg.TooBig { + logger.Warn("event with tooBig flag set") + } + if msg.Rebase { + logger.Warn("event with rebase flag set") + } + + commit, repo, err := LoadRepoFromCAR(ctx, bytes.NewReader([]byte(msg.Blocks))) + if err != nil { + return nil, err + } + + if commit.Rev != rev.String() { + return nil, fmt.Errorf("rev did not match commit") + } + if commit.DID != did.String() { + return nil, fmt.Errorf("rev did not match commit") + } + // TODO: check that commit CID matches root? re-compute? + + // load out all the records + for _, op := range msg.Ops { + if (op.Action == "create" || op.Action == "update") && op.Cid != nil { + c := (*cid.Cid)(op.Cid) + nsid, rkey, err := syntax.ParseRepoPath(op.Path) + if err != nil { + return nil, fmt.Errorf("invalid repo path in ops list: %w", err) + } + // don't use the returned bytes, but do actually read them out of store (not just CID) + _, val, err := repo.GetRecordBytes(ctx, nsid, rkey) + if err != nil { + return nil, err + } + if *c != *val { + return nil, fmt.Errorf("record op doesn't match MST tree value") + } + } + } + + // TODO: once firehose format is fully shipped, remove this + for _, o := range msg.Ops { + switch o.Action { + case "delete": + if o.Prev == nil { + logger.Info("can't invert legacy op", "action", o.Action) + return repo, nil + } + case "update": + if o.Prev == nil { + logger.Info("can't invert legacy op", "action", o.Action) + return repo, nil + } + } + } + + ops, err := parseCommitOps(msg.Ops) + if err != nil { + return nil, err + } + ops, err = NormalizeOps(ops) + if err != nil { + return nil, err + } + + invTree := repo.MST.Copy() + for _, op := range ops { + if err := InvertOp(&invTree, &op); err != nil { + // print the *non-inverted* tree + //mst.DebugPrintTree(repo.MST.Root, 0) + return nil, err + } + } + computed, err := invTree.RootCID() + if err != nil { + return nil, err + } + if msg.PrevData != nil { + c := (*cid.Cid)(msg.PrevData) + if *computed != *c { + return nil, fmt.Errorf("inverted tree root didn't match prevData") + } + logger.Debug("prevData matched", "prevData", c.String(), "computed", computed.String()) + } else { + logger.Info("prevData was null; skipping tree root check") + } + + return repo, nil +} + +func parseCommitOps(ops []*comatproto.SyncSubscribeRepos_RepoOp) ([]Operation, error) { + //out := make([]Operation, len(ops)) + out := []Operation{} + for _, rop := range ops { + switch rop.Action { + case "create": + if rop.Cid == nil || rop.Prev != nil { + return nil, fmt.Errorf("invalid repoOp: create") + } + op := Operation{ + Path: rop.Path, + Prev: nil, + Value: (*cid.Cid)(rop.Cid), + } + out = append(out, op) + case "delete": + if rop.Cid != nil || rop.Prev == nil { + return nil, fmt.Errorf("invalid repoOp: delete") + } + op := Operation{ + Path: rop.Path, + Prev: (*cid.Cid)(rop.Prev), + Value: nil, + } + out = append(out, op) + case "update": + if rop.Cid == nil || rop.Prev == nil { + return nil, fmt.Errorf("invalid repoOp: update") + } + op := Operation{ + Path: rop.Path, + Prev: (*cid.Cid)(rop.Prev), + Value: (*cid.Cid)(rop.Cid), + } + out = append(out, op) + default: + return nil, fmt.Errorf("invalid repoOp action: %s", rop.Action) + } + } + return out, nil +} + +func VerifySyncMessage(ctx context.Context, dir identity.Directory, msg *comatproto.SyncSubscribeRepos_Sync) (*Commit, error) { + return VerifyCommitSignatureFromCar(ctx, dir, []byte(msg.Blocks)) +} + +// temporary/experimental code showing how to verify a commit signature from firehose +// +// TODO: in real implementation, will want to merge this code with `VerifyCommitMessage` above, and have it hanging off some service struct with a configured `identity.Directory` +func VerifyCommitSignature(ctx context.Context, dir identity.Directory, msg *comatproto.SyncSubscribeRepos_Commit) error { + _, err := VerifyCommitSignatureFromCar(ctx, dir, []byte(msg.Blocks)) + return err +} + +func VerifyCommitSignatureFromCar(ctx context.Context, dir identity.Directory, car []byte) (*Commit, error) { + commit, _, err := LoadCommitFromCAR(ctx, bytes.NewReader(car)) + if err != nil { + return nil, err + } + + if err := commit.VerifyStructure(); err != nil { + return nil, err + } + + did, err := syntax.ParseDID(commit.DID) + if err != nil { + return nil, err + } + + ident, err := dir.LookupDID(ctx, did) + if err != nil { + return nil, err + } + pubkey, err := ident.PublicKey() + if err != nil { + return nil, err + } + + err = commit.VerifySignature(pubkey) + if err != nil { + return nil, err + } + return commit, nil +} diff --git a/atproto/repo/sync_test.go b/atproto/repo/sync_test.go new file mode 100644 index 000000000..313cf4ed5 --- /dev/null +++ b/atproto/repo/sync_test.go @@ -0,0 +1,56 @@ +package repo + +import ( + "bytes" + "context" + "encoding/json" + "os" + "testing" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/repo/mst" + + "github.com/stretchr/testify/assert" +) + +func TestFirehoseTrimTopPartial(t *testing.T) { + // "failed to invert op: can not prune top of tree: MST is not complete" + testCommitFile(t, "testdata/firehose_commit_4623075231.json") +} + +func TestFirehoseMergePartialNodes(t *testing.T) { + // "failed to invert op: can't merge partial nodes" (from bridgyfed PDS) + //testCommitFile(t, "testdata/firehose_commit_4621317030.json") + + // "failed to invert op: can't merge partial nodes" (from bridgyfed PDS) + //testCommitFile(t, "testdata/firehose_commit_4621317332.json") + + // "failed to invert op: can not merge child nodes: MST is not complete" (from bridgyfed PDS) + //testCommitFile(t, "testdata/firehose_commit_4621332152.json") +} + +func testCommitFile(t *testing.T, p string) { + assert := assert.New(t) + ctx := context.Background() + + body, err := os.ReadFile(p) + assert.NoError(err) + if err != nil { + t.Fail() + } + + var msg comatproto.SyncSubscribeRepos_Commit + if err := json.Unmarshal(body, &msg); err != nil { + t.Fail() + } + + _, err = VerifyCommitMessage(ctx, &msg) + assert.NoError(err) + if err != nil { + _, repo, err := LoadRepoFromCAR(ctx, bytes.NewReader([]byte(msg.Blocks))) + if err != nil { + t.Fail() + } + mst.DebugPrintTree(repo.MST.Root, 0) + } +} diff --git a/atproto/repo/testdata/commit-proof-fixtures.json b/atproto/repo/testdata/commit-proof-fixtures.json new file mode 100644 index 000000000..6de88d89e --- /dev/null +++ b/atproto/repo/testdata/commit-proof-fixtures.json @@ -0,0 +1,118 @@ +[ + { + "comment": "two deep split", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": [ + "A0/374913", + "B1/986427", + "C0/451630", + "E0/670489", + "F1/085263", + "G0/765327" + ], + "adds": ["D2/269196"], + "dels": [], + "rootBeforeCommit": "bafyreicraprx2xwnico4tuqir3ozsxpz46qkcpox3obf5bagicqwurghpy", + "rootAfterCommit": "bafyreihvay6pazw3dfa47u5d2tn3rd6pa57sr37bo5bqyvjuqc73ib65my", + "blocksInProof": [ + "bafyreieazvzmba35p4phksumwfoklwe5o4ncmo7otud74idcyv4orrbzxi", + "bafyreie4227qpa4vbtbpnsvuhp322b776vjuhxsidi5hxp2gawumr4m3de", + "bafyreid44jgimksqqdratyste2moqu6zo4h6co2pknjppfoiplsqxtuxae", + "bafyreiaerlvitye7fjjwodkshtbqqdsmfsdjtnlz4vs6y4trnddshsmd5a", + "bafyreihvay6pazw3dfa47u5d2tn3rd6pa57sr37bo5bqyvjuqc73ib65my" + ] + }, + { + "comment": "two deep leafless split", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/374913", "B0/601692", "D0/952776", "E0/670489"], + "adds": ["C2/014073"], + "dels": [], + "rootBeforeCommit": "bafyreialm5sgf7pijawbschsjpdevid5rss5ip3d4n4w6cc4mhu53sfl4i", + "rootAfterCommit": "bafyreibxh4iztp5l2yshz3ectg2qjpeyprpw2gogao3pvceowpq3k3thya", + "blocksInProof": [ + "bafyreih7dxytqtcjv3cfia3fi3wxofeip62teqkpynnkxisxqwfchfb4bu", + "bafyreiaqbymlnvpklmogx75gozjl3y73gva43jbgwcrqu2pp5g5ejou5vm", + "bafyreicfh3st5ghtnoqyyvznjv4lhfnvl7qsndempx35i4tcmoxakqbgrm", + "bafyreieyjrrai6igjceyxzkajrxgxz37da2eufb33anvesb4ev6yzztauu", + "bafyreibxh4iztp5l2yshz3ectg2qjpeyprpw2gogao3pvceowpq3k3thya" + ] + }, + { + "comment": "add on edge with neighbor two layers down", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/374913", "B2/827649", "C0/451630"], + "adds": ["D2/269196"], + "dels": [], + "rootBeforeCommit": "bafyreigc6ay2qwfk7kuevvrczummpd64nknfo4yxpaooknfymzyb7u3ntq", + "rootAfterCommit": "bafyreign6kxoll35r5f2ske6hjx7vg56aw3jn6r5hcopgrepzafpvohr2a", + "blocksInProof": [ + "bafyreieazvzmba35p4phksumwfoklwe5o4ncmo7otud74idcyv4orrbzxi", + "bafyreidicvcjgrpm5bmhm3ndh2ysqfhgzk4chwn3m4kuvwkenfusspb4uy", + "bafyreign6kxoll35r5f2ske6hjx7vg56aw3jn6r5hcopgrepzafpvohr2a" + ] + }, + { + "comment": "merge and split in multi-op commit", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/374913", "B2/827649", "D2/269196", "E0/670489"], + "adds": ["C2/014073"], + "dels": ["B2/827649", "D2/269196"], + "rootBeforeCommit": "bafyreiceld4icym4qjmdcn3dfgtxt7t66hdgyhvigessgmkvb56dx6amgi", + "rootAfterCommit": "bafyreigkalika3taqauapfha556lo36zzcjoiifny5xeru6yis3nxw5ruq", + "blocksInProof": [ + "bafyreid44jgimksqqdratyste2moqu6zo4h6co2pknjppfoiplsqxtuxae", + "bafyreihytu6onh476trave25zuo63ziebkeong2755sc5nmf55uzdawgt4", + "bafyreigkalika3taqauapfha556lo36zzcjoiifny5xeru6yis3nxw5ruq", + "bafyreidnnkrdkcaswbflgtdsxm7nzs7p5f2rdous6wrlupzstuwqu5pfgm", + "bafyreia2kq243hqq3volwlzkbzzphoeqauk54sc5h7vgogq4ei5fjizxvy" + ] + }, + { + "comment": "complex multi-op commit", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": [ + "B0/601692", + "C2/014073", + "D0/952776", + "E2/819540", + "F0/697858", + "H0/131238" + ], + "adds": ["A2/827942", "G2/611528"], + "dels": ["C2/014073"], + "rootBeforeCommit": "bafyreigr3plnts7dax6yokvinbhcqpyicdfgg6npvvyx6okc5jo55slfqi", + "rootAfterCommit": "bafyreiftrcrbhrwmi37u4egedlg56gk3jeh3tvmqvwgowoifuklfysyx54", + "blocksInProof": [ + "bafyreih62n3gjbzzvlicuggpfydyzrp3ssyx7hdgtltd3sct3ribm3u73e", + "bafyreihrjhuoynjvgteuefin5vwnqmupyfzvmytdobpstqt3mbawgw5qhm", + "bafyreibevzst4gzkxo263syohlmq3lpxdvpjhlpyqx2ay3moh43lifydca", + "bafyreifsdd7dv2neal7zjhyrsvndkaocelqlpgfxwo4utoq2g77klih37e", + "bafyreid2wwyroodj2lxx2obikac74q77lsn6vqkoetlqqwwnr3criwlcvy", + "bafyreie55b224oljhykpsxdjq4ajn2ysksud7qm347s6kn2ei6a775faum", + "bafyreiftrcrbhrwmi37u4egedlg56gk3jeh3tvmqvwgowoifuklfysyx54" + ] + }, + { + "comment": "split with earlier leaves on same layer", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": [ + "app.bsky.feed.post/3lo3kqqljmfe2", + "app.bsky.feed.post/3log4547dm6h2", + "app.bsky.feed.post/3log45inogon2", + "app.bsky.feed.post/3logaodrh74d2", + "app.bsky.feed.post/3logteazog2n2", + "app.bsky.feed.post/3lon5cqsbwrj2", + "app.bsky.feed.repost/3l6sjhvqonco2" + ], + "adds": ["app.bsky.feed.post/3lon5dzeaihj2"], + "dels": [], + "rootBeforeCommit": "bafyreigfcsro2up7qi7l3rxdpg7n6gjtteotkmgrrqztl5oy2tf4ncl4ji", + "rootAfterCommit": "bafyreig33hsjiplaixvmccy65n7rn3in5nsbtcittzx6k3w5wjfhk2sg3a", + "blocksInProof": [ + "bafyreig33hsjiplaixvmccy65n7rn3in5nsbtcittzx6k3w5wjfhk2sg3a", + "bafyreih2rhjm3apcghihwfojv2em7noqkgt5qyjcnxux7do674m464oc3m", + "bafyreiajhswkduap4zvqvfhth3skdgckmk2eb5gow7vv3gvj45f4fqwmxm" + ] + } +] diff --git a/atproto/repo/testdata/firehose_commit_4621317030.json b/atproto/repo/testdata/firehose_commit_4621317030.json new file mode 100644 index 000000000..4fcb45631 --- /dev/null +++ b/atproto/repo/testdata/firehose_commit_4621317030.json @@ -0,0 +1,26 @@ +{ + "blobs": null, + "blocks": { + "$bytes": "OqJlcm9vdHOB2CpYJQABcRIgP888UjM8r4WrcB33FNH3KFMI7iDXzM/vyvckq33gb6BndmVyc2lvbgGpBwFxEiAc+J+iVgZraHEQH5doUEVvRqSOlf+I2n+fr2GcSVSeG6JhZYikYWtYImFwcC5ic2t5LmZlZWQucmVwb3N0LzNsZ2RlMm13dGxjcjJhcABhdNgqWCUAAXESIIEUP1lIUlLfHBVsLleTz+xmgkZglHMvQnbEBP341+GNYXbYKlglAAFxEiDKqWxSs3JmWtSR9IhwVdGFJVJWi4QjMyDZmpB/Ievu16Rha0poNzVxZG5keHMyYXAYGGF02CpYJQABcRIgB1U2OmourFJ/FdLXmmn2A4r8TEGam+ZvVlS6GlG7ZMNhdtgqWCUAAXESIK6gegsvplIyBSBFOs23wbA6xmfdDDrvrbOS+YZgegampGFrSXpkdjVjNTRzMmFwGBlhdNgqWCUAAXESIJIueCeRzc02ySC11d+m5k8kpQbqSvob8sejybMXCHgOYXbYKlglAAFxEiDu3D0atZWDfAldw7lkCmX4oaHW1dCv4XfbuzJuPUURpKRha0pzaDU3dzdsbzQyYXAYGGF02CpYJQABcRIgDPMhUJORDRvy4DM5iYwYkA8ctbcYs590EqBBUFs3vHBhdtgqWCUAAXESIH0FBYVlUBCDS7XMmHkLsIS/+uI+Vej+3L0OUJmONY6FpGFrS2hibDVyZWFpcXEyYXAXYXTYKlglAAFxEiC19nv3O98fDIPrvbAbUVMP8fgoFrjjwmapi6MtcZGLB2F22CpYJQABcRIgbHGyY+za1M1qc457pounPXfp2Dc43OkZAORC45LgY12kYWtJbXVhdHJncGcyYXAYGWF02CpYJQABcRIgQZcrUipYp7MoQ49h5GBXOfRKzCt5mfPLPIHk+4klY6VhdtgqWCUAAXESIGhu0I3PF4eNnIMUhZ3tAzoJ5FuweXT4RFF3xZufvX2dpGFrSmNycGJsaG40dTJhcBgYYXTYKlglAAFxEiCeGxEE3mRFhaL9AYYgHqWg78wU9/gPay6Z06NuvLshm2F22CpYJQABcRIgzRsw/79cg7A9bDRUecE6SWiS8w9Tr/KM1ymCjcsc0NKkYWtKaDJ5NGZydWEyMmFwGBhhdNgqWCUAAXESIFL7PABMCQLiF8/5iS4UkyfQcUt1WmULGaHNu+oCLBiZYXbYKlglAAFxEiAo0XYvxnDQ3MwjT7UIa00o8ZBwfnHoXvBUi/EVt4O8smFs2CpYJQABcRIgMRUbaaMo+bRXKVZnDh4up4Q3b/DeBuxHkltNEThLsHz8AwFxEiAGTK6c8kNeQHEZ8hSkoXQ1RmQ30EgTg4S3dxXTp860iaJhZYekYWtYImFwcC5ic2t5LmZlZWQucmVwb3N0LzNsaGhhaGc3YTJyZjJhcABhdPZhdtgqWCUAAXESICKfP2leB3LRPMcx9vttm8+d6S8CJ5+nJP2HCNtifCZgpGFrSWQyb2M1aXlmMmFwGBlhdPZhdtgqWCUAAXESIB2+fDGtuCFUKavYxz367LzQDNbhBR+2OIopxKvZulr3pGFrSHVmbzR2dTIyYXAYGmF09mF22CpYJQABcRIgQBO0qbeJOoXgS9p+U/x4pEZTzbh1pskmb1Zx5t9s/2+kYWtJZmM2cWEyN2EyYXAYGWF09mF22CpYJQABcRIgPzeWgmN+u0moUHbzM/d5ZY381u+2eEGCxmW6Vy9W/PmkYWtJaHpuMmI2bWEyYXAYGWF09mF22CpYJQABcRIgXrcBCY7rcfcROmiPGdpineroI8zQmhnpClD66jAkmwakYWtJc2dob2JyaXIyYXAYGWF09mF22CpYJQABcRIgtJN6IIazO1P3n/wfA9cGv5LDLUllrcvQHyDhaAfzXhekYWtJdTMydDY2d2QyYXAYGWF09mF22CpYJQABcRIgIdjWvACMt+nQP4YawgRsY9SE82IF4EMO87VtGg1rUyFhbPaIAgFxEiA/zzxSMzyvhatwHfcU0fcoUwjuINfMz+/K9ySrfeBvoKZjZGlkeCBkaWQ6cGxjOjZoeTVwNnJtM3psY2RlcGhxc3NjMnA3NmNyZXZtMjIyMjIyZDduN2kyMmNzaWdYQLUY+pgU+E9glNaSASgEbTnPnR9O2dETHA6sxuq29YSuM27K6MdHEavJqT9tcx1ikxLY4BB6pQV/+I4wVfgg+GxkZGF0YdgqWCUAAXESIL8NojUibe0s+GdHFpx5GvGz3XJQQMWHioEKqpbc5OEUZHByZXbYKlglAAFxEiCUs9ALVpzzdpyQut7buZ4B7FXlXSTkdS3MR6Qtv+O8lWd2ZXJzaW9uA7sCAXESIDTMmJQyN3NA5I56qg4fiNfI3q91iQ9mZ6O5anSLRSWwomFlgqRha1giYXBwLmJza3kuZmVlZC5yZXBvc3QvM2xmeGdnN3drZmNqMmFwAGF02CpYJQABcRIg/7sexA1qhKY2vKGsJ8K3Fo0mWKOmhQxhwq6fxYRpXLdhdtgqWCUAAXESIFqJy3ug1RiBvsRYnBEi3eifF3WiTlnie8N9A8UKkDW7pGFrS2cybG1naGhtbnMyYXAXYXTYKlglAAFxEiAc+J+iVgZraHEQH5doUEVvRqSOlf+I2n+fr2GcSVSeG2F22CpYJQABcRIgCrXGp1R02Nx0+3tUKfVVu9XXTnwUqPHnKRl8D1qk7F1hbNgqWCUAAXESICZl8BPys/ta7ir0kiZ3OOyqzbRdehvHzP76cSVzL895UwFxEiBS+zwATAkC4hfP+YkuFJMn0HFLdVplCxmhzbvqAiwYmaJhZYBhbNgqWCUAAXESINqNdN12WUQ9TMLvHNTP6/xVTYUamApfu0WiR/NXL6mC0wEBcRIgvw2iNSJt7Sz4Z0cWnHka8bPdclBAxYeKgQqqltzk4RSiYWWBpGFrWCJhcHAuYnNreS5mZWVkLnJlcG9zdC8zbGZhY21zZ3ZvYmcyYXAAYXTYKlglAAFxEiA0zJiUMjdzQOSOeqoOH4jXyN6vdYkPZmejuWp0i0UlsGF22CpYJQABcRIg2DtT7U6/IEZ7A5PzoY94mr3ADUnxrpFbULVSv662bthhbNgqWCUAAXESICtBA1kz0QeiYkOUGhfF007JqZsjTvh757cWZZS/T3na+gEBcRIgooFn0kuPCY/VLzN6dXsUIBboGYdkPeYY0S5GhXaAcEKjZSR0eXBldGFwcC5ic2t5LmZlZWQucmVwb3N0Z3N1YmplY3SiY2NpZHg7YmFmeXJlaWg2NXJtdmR1cWxiM2hramw2YjRzcjYya3hxbHdrNTJ6c2M1c2g2aHJlcng3amF4eWoyZjRjdXJpeEZhdDovL2RpZDpwbGM6NWYyc29jaHZ6cGtjNnBvZXRlbTU0bjZ0L2FwcC5ic2t5LmZlZWQucG9zdC8zbGhodjVsYXF0dXIyaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowMDo0Ni4wMDBa0wEBcRIg2o103XZZRD1Mwu8c1M/r/FVNhRqYCl+7RaJH81cvqYKiYWWBpGFrWCJhcHAuYnNreS5mZWVkLnJlcG9zdC8zbGhodmRrY2J5M2UyYXAAYXTYKlglAAFxEiA93F4NaTv+XPo+6v7gIFbWlrzStja20QTE8c1FAte8GWF22CpYJQABcRIgooFn0kuPCY/VLzN6dXsUIBboGYdkPeYY0S5GhXaAcEJhbNgqWCUAAXESIAZMrpzyQ15AcRnyFKShdDVGZDfQSBODhLd3FdOnzrSJ" + }, + "commit": { + "$link": "bafyreib7z46femz4v6c2w4a564knd5zikmeo4igxzth67sxxesvx3ydpua" + }, + "ops": [ + { + "action": "create", + "cid": { + "$link": "bafyreifcqft5es4pbgh5klztpj2xwfbac3ubtb3ehxtbrujoi2cxnadqii" + }, + "path": "app.bsky.feed.repost/3lhhvdkcby3e2" + } + ], + "prev": null, + "rebase": false, + "repo": "did:plc:6hy5p6rm3zlcdephqssc2p76", + "rev": "222222d7n7i22", + "seq": 4621317030, + "since": null, + "time": "2025-02-06T01:05:06.528Z", + "tooBig": false +} \ No newline at end of file diff --git a/atproto/repo/testdata/firehose_commit_4621317332.json b/atproto/repo/testdata/firehose_commit_4621317332.json new file mode 100644 index 000000000..dd4dd85fe --- /dev/null +++ b/atproto/repo/testdata/firehose_commit_4621317332.json @@ -0,0 +1,26 @@ +{ + "blobs": null, + "blocks": { + "$bytes": "OqJlcm9vdHOB2CpYJQABcRIgQ8oa2D2MRn6uWUgg0pK9eHDHmx8C/NT3fM26aG33qHhndmVyc2lvbgG5BwFxEiAmJ11tROjXcYIwTe1+NxaoCduQsriUcucvVhF1v19YoaJhZYikYWtYIGFwcC5ic2t5LmZlZWQubGlrZS8zbGF2Z3hhaTdrbHgyYXAAYXTYKlglAAFxEiB7nKQMYr8S2RKgG77E1ktNNDscq4NuP+w+lkZH7HNwbmF22CpYJQABcRIgPYEa4DK+G7QOmGyu0NzQzVEHWDvl3OxkpxoumQJyCzikYWtLZHNrcmRhYmozbjJhcBVhdNgqWCUAAXESIHN58X8+mERHBS2Hdcooxil07G1qgifhnc7ppkHJI0ZHYXbYKlglAAFxEiDXBvn1o/TWY6C/C2QDdxgjkPsNne639o8AsKDkgJIKGqRha1Jwb3N0LzNsNmJ4d28yMjdqbDJhcA5hdNgqWCUAAXESIIgb6WQkFP1kiJH87p3vTSbkOTZ0GxRdho8WAOnE2QMZYXbYKlglAAFxEiDPucd1mIDr+qoZpt9h31GlNlgazN24MEfTfcOoZuHstqRha0s3cmgyNHlxa3duMmFwFWF02CpYJQABcRIgVn6eOdzuM4wnxy+BGnVnHuLiOxmrhQ+jgS0OxovBrIFhdtgqWCUAAXESIDXud3kQG+U7aulDJl6bILVKOshPlnTXIV4tGyvEyuHupGFrS2NocXBqc2dkdHYyYXAVYXTYKlglAAFxEiBLlUua5lulVaXG75B/2fkXzNLHvxQI/TL3dfNca9etomF22CpYJQABcRIgaD388Uag11Z0cuLgGFYerUD+E6kSrDuZ9ck3S0hS1YukYWtLZWJueXo3am4yaTJhcBVhdNgqWCUAAXESIMOBqmuxCawNBt+QoVRZRfIBUzHF1abB9bz6q+49K8AMYXbYKlglAAFxEiAJdL4BIl+K9CcQVpCo5CttdJYBLUbpjdmtnR2mRoZFZaRha1RyZXBvc3QvM2w1dmQzbnk2Y2t4MmFwDmF02CpYJQABcRIgHz/ccWVwXohz21ERZsNK2UWNAKyYdI5mpC/XL1M+KFFhdtgqWCUAAXESINAFjz4dJIx8vO7n+afe+kF1Ap7mxxv7y+fulfD21J1spGFrS2FwcjQ1bXpwbTYyYXAXYXTYKlglAAFxEiDbVHRnfni6OyrSAv+8M3ohZgHNkOjD/LDjrmiPwxiRuWF22CpYJQABcRIgyGEh9U8hCOnL1aiNPtj7cPUSojBUywEnoduN1KvWh5lhbNgqWCUAAXESIC0XcgxVa1VxXydTYqnO7i0qKsKXXF+UgbvGJO/NdNvMiAIBcRIgQ8oa2D2MRn6uWUgg0pK9eHDHmx8C/NT3fM26aG33qHimY2RpZHggZGlkOnBsYzoyYnBib2ViNnd0Mnd0dnFwZ3ZnZ2ptbDZjcmV2bTIyMjIyMmQ3bjdxMjJjc2lnWEDLpAzW9FqZ2fYH+/Fh8hoCwiVDaif+4EofQ2Wl8wBDqQEHmHkHIsn8w5xMifRC7cmjZ1SAlMDJOcZPOkr4ThteZGRhdGHYKlglAAFxEiAmJ11tROjXcYIwTe1+NxaoCduQsriUcucvVhF1v19YoWRwcmV22CpYJQABcRIgraxiJ1FSWkO/r1saRqysNBcyyo6aiWwoZ8Gs/TX0aDNndmVyc2lvbgP6AQFxEiCbllVgVYWJO6FLGQTbX5evo7pipBZvChj2GynsCJR98KNlJHR5cGV0YXBwLmJza3kuZmVlZC5yZXBvc3Rnc3ViamVjdKJjY2lkeDtiYWZ5cmVpZWh3bzJ3aXYzMzI1dnI3czJ5bG1ldXNmZnBia2JreXRhaG5kem9mbnBpd3JpbXBleWd0aWN1cml4RmF0Oi8vZGlkOnBsYzoyYnBib2ViNnd0Mnd0dnFwZ3ZnZ2ptbDYvYXBwLmJza3kuZmVlZC5wb3N0LzNsYjVyYzRjZGJyNDJpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA0OjI3LjQ1M1qTAgFxEiCGy20I5h9gTuiSz6XQ51lPLT3fXPGtIvqUhkf48i9k+6JhZYKkYWtYImFwcC5ic2t5LmZlZWQucmVwb3N0LzNsaGZjZGZsNzNnbDJhcABhdNgqWCUAAXESIMiGBX2KQ0qazPI1kCC7weh9GE22TviKjG4oulGUngVfYXbYKlglAAFxEiAd9kPS4SQoBR6nAB2yb+bksh5j/qdCxHP1/Gc5TBmb2aRha0podmRyNjJjemUyYXAYGGF02CpYJQABcRIg4FG08M+BIcLzxeK+6dPnpMohINBnWWThg6QAfHRdeyBhdtgqWCUAAXESIJuWVWBVhYk7oUsZBNtfl6+jumKkFm8KGPYbKewIlH3wYWz2uwIBcRIg21R0Z354ujsq0gL/vDN6IWYBzZDow/yw465oj8MYkbmiYWWCpGFrWCJhcHAuYnNreS5mZWVkLnJlcG9zdC8zbGFxNHpsZTM3ZmUyYXAAYXTYKlglAAFxEiAS8xuL6Rv0H2LJgLOxfGHRwhuY6APrWrC1v8XWcpYTYmF22CpYJQABcRIgOThGBVW5Pfd8UWem0u+ZAgl2pINbS5YZQ+kPjmOo4oykYWtLaGZjN3dkbmNjcDJhcBdhdNgqWCUAAXESIPPOGPP2H1OFkKhgwj4WZe0kvXoLlHIqRNfP2XHTcBImYXbYKlglAAFxEiADxis9NT3gkmlHv2MAsQoylgYTSDYKaHPCyIEOEzGRL2Fs2CpYJQABcRIgoxLvwh0tdq9swgX1gR4ms4OF9L6rObRAFY/nSoehe8XDAQFxEiDIhgV9ikNKmszyNZAgu8HofRhNtk74ioxuKLpRlJ4FX6JhZYKkYWtYImFwcC5ic2t5LmZlZWQucmVwb3N0LzNsaGZlNWYyc3lwcDJhcABhdPZhdtgqWCUAAXESIOkEqXYZUo6+G5V5QDnkSVqvnM8qKC9hnaJI5x0ppxrCpGFrSmhzN2xud2t1NzJhcBgYYXT2YXbYKlglAAFxEiCt6w69VSxF1+sZ26TSbxbrm+2AD910k0MgmD2P7gUEemFs9lMBcRIg884Y8/YfU4WQqGDCPhZl7SS9eguUcipE18/ZcdNwEiaiYWWAYWzYKlglAAFxEiCGy20I5h9gTuiSz6XQ51lPLT3fXPGtIvqUhkf48i9k+w" + }, + "commit": { + "$link": "bafyreicdzinnqpmmiz7k4wkiedjjfplyoddzwhyc7tkpo7gnxjug355ipa" + }, + "ops": [ + { + "action": "create", + "cid": { + "$link": "bafyreie3szkwavmfre52csyzatnv7f5puo5gfjawn4fbr5q3fhwarfd56a" + }, + "path": "app.bsky.feed.repost/3lhhvdr62cze2" + } + ], + "prev": null, + "rebase": false, + "repo": "did:plc:2bpboeb6wt2wtvqpgvggjml6", + "rev": "222222d7n7q22", + "seq": 4621317332, + "since": null, + "time": "2025-02-06T01:05:06.935Z", + "tooBig": false +} \ No newline at end of file diff --git a/atproto/repo/testdata/firehose_commit_4621332152.json b/atproto/repo/testdata/firehose_commit_4621332152.json new file mode 100644 index 000000000..f46fe20e1 --- /dev/null +++ b/atproto/repo/testdata/firehose_commit_4621332152.json @@ -0,0 +1,313 @@ +{ + "blobs": null, + "blocks": { + "$bytes": "OqJlcm9vdHOB2CpYJQABcRIgqyrRIy8N/q+LfGxBwJ73EbEMeli2qptGcFrD2v54xjtndmVyc2lvbgHRAQFxEiB7HtprOYF6LHXoeF6r7Cr2qMG9mPb/M1TmUemZZBSuiKJhZYGkYWtYIGFwcC5ic2t5LmZlZWQubGlrZS8zbGF2dDV4cHBneTJ4YXAAYXTYKlglAAFxEiCn3KXqJzS2kKuFV5IO4oNxPhbV4nMAlasUKIVrt4mEiWF22CpYJQABcRIggVdCYKWwmOZPLVzbHVqfmeU/AQH5ZfUWAHAMpXYtUUZhbNgqWCUAAXESICU8pmcWsWl/M+oquFdFaozMCSJb/g8u9c5V0KSkZkDsUwFxEiCn3KXqJzS2kKuFV5IO4oNxPhbV4nMAlasUKIVrt4mEiaJhZYBhbNgqWCUAAXESIDxGFJEDxK5+SBLFWdlLJGYo7xCgxlJ2bKqBR+cCYd3ZUwFxEiA8RhSRA8SufkgSxVnZSyRmKO8QoMZSdmyqgUfnAmHd2aJhZYBhbNgqWCUAAXESILCRoej+qP2JXO0HvPhoqllexAudtaNmf8PMfyHWZqk9UwFxEiCwkaHo/qj9iVztB7z4aKpZXsQLnbWjZn/DzH8h1mapPaJhZYBhbNgqWCUAAXESION+97kPdHvJROaWmxVqIwy8i9bVzoLjNbVH3+pDZwX8UwFxEiDjfve5D3R7yUTmlpsVaiMMvIvW1c6C4zW1R9/qQ2cF/KJhZYBhbNgqWCUAAXESIPzbk1kmpvmyLvuSkv3Nt+LXiaO0J/qZwTiIz1Hl4YydUwFxEiD825NZJqb5si77kpL9zbfi14mjtCf6mcE4iM9R5eGMnaJhZYBhbNgqWCUAAXESIAVYW8VyIBP+U3LmMm7Nwi3v2nim45/+YZxFprw8OMekkgQBcRIgBVhbxXIgE/5TcuYybs3CLe/aeKbjn/5hnEWmvDw4x6SiYWWEpGFrWCBhcHAuYnNreS5mZWVkLmxpa2UvM2xidm8ybndrbWsyaGFwAGF02CpYJQABcRIg4fjEKsds0bi9d53csvBBiMwgemp8KkWERpotKTeeahVhdtgqWCUAAXESIAha0iqGuicbJCynZzsj0OVKN962VyNKkO+4R1e46RizpGFrS2NuNmN0aGp3ajJyYXAVYXTYKlglAAFxEiAHYChmR8wz2Vl+ijHeW9H9KxEgqZQt1zyy6f/Ee9hIEmF22CpYJQABcRIgkWu4sncXFJ2X9vLDTHk9U7wOXBNon5qhyX1iXvDBzWukYWtUcmVwb3N0LzNsYmUyZmh3c3B1MmVhcA5hdNgqWCUAAXESILe26/MVnf7GhwJlvV1q2+zaZzsdJcr9Sx9uGac5z/8XYXbYKlglAAFxEiD8uvxjctf17+bZjdwD3d0Fspbfzs08kWwYe0e49OyKwaRha0tjYWs1c3VsZGQyZmFwF2F02CpYJQABcRIgv/RdKBLzVXiAvYsRofWyR5S9nZ6NgIEmM995yrtwNFJhdtgqWCUAAXESIJSpS6Vf5eet04HvcSzGionoxZB0On601Cddl8mHiW0GYWzYKlglAAFxEiDUj9dj9ALxgq6JQvN44lL1Ls+LokbQtztEiOy57xx/B8QGAXESIL/0XSgS81V4gL2LEaH1skeUvZ2ejYCBJjPfecq7cDRSomFlh6Rha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsY2g1ZGt3Z3ZhMnNhcABhdNgqWCUAAXESIPy110h+INyVC2DAysIrvZHeI14BMR0XU1inejE3GqUQYXbYKlglAAFxEiCfu26FT7X1ss4ENufQfuaDHCyAgyH2pW0gUNE/+5i/baRha0tkZzQyanpjbXIyY2FwGBhhdNgqWCUAAXESINp9vUDilnFjGzUzroYdNf7VnUecoCaDVTEhlquwaSUgYXbYKlglAAFxEiBzCgHD5mXNan5XkNVUrjFb6oBTdqWwqv0cZnk1b7skRKRha0tmemJpeHFmY2gyaGFwGBhhdNgqWCUAAXESIHpBXTaOZzjmVcACXm0gCGc6NP6I30O1ShHqSRP0BLWlYXbYKlglAAFxEiA543K72+4fL4H2KT7OHGohg1YmAFaKjZdwfEbkdXc8eaRha0toNXg1MzVoZWcyamFwGBhhdNgqWCUAAXESIJa4zeBsWNl1/9i/NvbGfYCRLX9vpmOjJor/aytUr/nqYXbYKlglAAFxEiD138d1GcEhM8qsjP34y8/2FCO0PesBxF6oN0pkTyEQBqRha0hlbWIzeWYyamFwGBthdNgqWCUAAXESIN/NvGm4vsRayF8rJZRKTXXL47nymVBiO/Ozz4lL0k3nYXbYKlglAAFxEiByOgKXhIidsf9hL6IPoV2UcNnDtGJPmiQUhulqJKgeNKRha0l5a2FxeW1zMmphcBgaYXTYKlglAAFxEiCyh9I4eT7hq0kXEZuapWM6n+YXmOwyFs7LzsegdQWdy2F22CpYJQABcRIgq2PMxoImcjIDTM8nBba9gco8tkr5z82MF5rMPo2EEOGkYWtKaG0zeDdoM2wyM2FwGBlhdNgqWCUAAXESIBZBOA0aRagsPHXcpQ063d/HafZyfxP7JWK+B1/V8oNGYXbYKlglAAFxEiAl9bvxG1MKFf1pmZ4mTaxh0JAvIxELDHMULZXIU6rSuWFs2CpYJQABcRIgGdjmbAsOJ3L7a53UI3I6zDX3/oH38BcTXqsz9nPVF0S6AgFxEiAWQTgNGkWoLDx13KUNOt3fx2n2cn8T+yVivgdf1fKDRqJhZYKkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhobTRxdnZqejIzYXAAYXTYKlglAAFxEiDbsUEc/nr4X7ya2hvr1Y3irqiWDf8zTfakGSATgXJe7GF22CpYJQABcRIgeJ5xdgqQ+6ISpCK7SJCHQky/PSpEcPw4cLUBJVdY2hCkYWtINmZwcmJsMjNhcBgbYXTYKlglAAFxEiAtBd9ZmUKgqUdvJDf+Ltc+pBiRa+tFxYx6Q3qrBWjsfmF22CpYJQABcRIgJa3L5QSN3LCIBTzxq/ZEZNkJWtEY8Uv37u1apjmbrethbNgqWCUAAXESINrsU4cmgwU58wIZGROSQWznR+s7syr3BSyoSiIQdNJ/hAQBcRIgLQXfWZlCoKlHbyQ3/i7XPqQYkWvrRcWMekN6qwVo7H6iYWWEpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaG02cjI1ejcyM2FwAGF02CpYJQABcRIgcvYdsIaAyuupqSVCtxWFW6+4RzWhApxDatVC5aGpOmRhdtgqWCUAAXESID01OUUZ3vZyjBiaYXvSqb0r7G9AtfxrYbhEz1j5Th+dpGFrSXZkYXM3Y2MyZGFwGBphdNgqWCUAAXESICXw93sInV2H4Rh9L09SedV2viynvZ/E76tCpaQSmLT3YXbYKlglAAFxEiASrc+UR53GpkvVKyOHh0lFJ2MvtqiJo/yDIUJeQ24P5KRha0hmYTJkdnIyZGFwGBthdNgqWCUAAXESICaGiRUv7PfcnZb2syvHKrjAzvjqTLdk5B1/gbQgYODuYXbYKlglAAFxEiAvkktjKZIgPXWh1NR5k/jeWvtIkjIujh3Vr7OZv71VJaRha0VldnEyZGFwGB5hdNgqWCUAAXESIOuIulptg0IlLDvGagKw1Ygb7PfWTBmraN13e2DjVSZAYXbYKlglAAFxEiA7rSUTFHrXXjiSi/phzGJF51NRYTavgnvTg5OJwUoCsmFs2CpYJQABcRIgFrNRgXtGAx45PzUyRqeUjwQUnNu/ZGw7glhndkRFZTDrBgFxEiAl8Pd7CJ1dh+EYfS9PUnnVdr4sp72fxO+rQqWkEpi096JhZYmkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhodmVkZDJydzJkYXAAYXTYKlglAAFxEiAheUAq+ZxU5ERvTfYr7we4l+h+empJCpYOXneaJKgGGmF22CpYJQABcRIgtOUTgOaLffVPfIcdEtE8IjdzPSN9JALAiUbJuvAHWm6kYWtEc3EyZGFwGB9hdNgqWCUAAXESILAUiZvCqqs4a5fKxX/uXmc65wNilamSfB4kd1W8dzHwYXbYKlglAAFxEiAIeVfXH31Tj1zIFO9+gQ78NGvrrC3r1cYXdR+quGIPjaRha0N1MmRhcBggYXTYKlglAAFxEiBofsFZzD3l7+7pJ8Vht6DFRHTgRoY1YCsuZvmkE85JQWF22CpYJQABcRIg1VYYiYeOaUCCme7mj5p/zra/kihBJ3/QN7vW60WQiOykYWtIZmEyZHUzMmRhcBgbYXT2YXbYKlglAAFxEiDiTn7olF8kc4SW0aHU4/WLuF/QHVqcMxZ1mtTvJNTAJKRha0M0MmRhcBggYXT2YXbYKlglAAFxEiAYIwvZlE/7MTNcT/rDmSX93TDZWBAJ+SA3gRmd3O6xQKRha0M1MmRhcBggYXTYKlglAAFxEiCjh72IEquowBfbMlvm7n7QS7ccMo+gptVGLXuMCpEpRWF22CpYJQABcRIgDrJUNrZi59QMFQ3ziGIdQiF/u6RcFJML5f6TnOet9rakYWtDdDJkYXAYIGF02CpYJQABcRIgDnaBytrb0Cf3LALW8kCabBJaTE5WaNw80kFaeAvLVeNhdtgqWCUAAXESIHhBqCB/VdnnxDK+WiW/qKh66NASmkClYOHPnK7NvrCJpGFrRHZlMmRhcBgfYXTYKlglAAFxEiBPLYIlZh2D28+8h9zRkHc9z2dpap+JPQOzxQ3zVNd26GF22CpYJQABcRIgGAT8Lj+7FASDQH3lChR7gu8iWA2XsTGBp8P374gzKFqkYWtDcTJkYXAYIGF09mF22CpYJQABcRIguobkhEUYnGlqua7TEdUJWYmki/0nSOi6Ob3lc4R7VklhbNgqWCUAAXESIAtPwTPeRjzSnuGqyePXTMF9OTWg6J6Z8DU2UlLdZbSolwMBcRIgDnaBytrb0Cf3LALW8kCabBJaTE5WaNw80kFaeAvLVeOiYWWDpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkdXYyZGFwAGF02CpYJQABcRIgMfbl8i5vYToWd47b/fUNZQ+hZD0MsI/GQRSUfLJrl7hhdtgqWCUAAXESIG/WrPmMhqGueDxV7vmstMkbMqgh/kkhDm8f4WV37kNdpGFrQ3gyZGFwGCBhdNgqWCUAAXESIFWilT9uxaUobragznpL+6o5LS7TnIUYgjsSkkHPO7WgYXbYKlglAAFxEiCOL07eR2fhQAS2CsQLetVflMMP4y2y33TZI0EImrIU0aRha0R2MzJkYXAYH2F02CpYJQABcRIg1sum48jfs2mvgLFayltsTWPIYy/1T0VcINNHlJ9SG/phdtgqWCUAAXESIM8F4y9uaZt9feX7wZofHLzu8GSF6BagiEv9zTXxLAOuYWzYKlglAAFxEiASbsjS5ZIKNZYVBu/x9pbNaxwSFmTMVJtNKlu97RyN5u4CAXESIE8tgiVmHYPbz7yH3NGQdz3PZ2lqn4k9A7PFDfNU13boomFlg6Rha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZHZsMmRhcABhdNgqWCUAAXESIEdv+kGtEVU6Kw/ws1AVZYzwNGxhOTIBJo2i5tuLqr4PYXbYKlglAAFxEiA9DX2X9W9NmUPLNMDYm9iqgkj2QR4zds9UESYnKeZHoqRha0NuMmRhcBggYXTYKlglAAFxEiCA7baB/wB5PaewguS8ufYN46DcGjWafRbrn9AWN0D4QGF22CpYJQABcRIgVP1y/spMhQ6mhKLdhIeaOU+vdek2ZL0WY4h0xcwT1yOkYWtDcDJkYXAYIGF09mF22CpYJQABcRIgZ5yPaeiaD2gGyXStXvJc+qo/21OAOmDxDXMy/M8S5NxhbNgqWCUAAXESILhQtI8q9tnZzfAJDvpHAGtJ5kNSWFfDZXT1JWlts46BoQMBcRIguFC0jyr22dnN8AkO+kcAa0nmQ1JYV8NldPUlaW2zjoGiYWWGpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkdmYyZGFwAGF09mF22CpYJQABcRIgLGJ77hDuLN9DiugWUd1zLbn+k4vBGxzlnUYn+h8OQlikYWtDZzJkYXAYIGF09mF22CpYJQABcRIgP8ymrMC9TCLHav8w6LrbqTQF/Oy6l20lSNz2/puYMA2kYWtDaDJkYXAYIGF09mF22CpYJQABcRIg1nT+bQT6khT1qnxFcxUvrcxQI2e8Yl0G0BLqrMYx6rukYWtDaTJkYXAYIGF09mF22CpYJQABcRIgJNoIf2FFjPrbocRc80TC2J81DZ8cr6e3JnLXlz8hL/qkYWtDajJkYXAYIGF09mF22CpYJQABcRIgxY6aDMRN3Imjc9uA3vrRQnI+EwHaJ/rV1+tTvecg2likYWtDazJkYXAYIGF09mF22CpYJQABcRIgvEmmV6xSPnOqE6e4eJ0zQ+mCmr/7fQf5YJ1ToQDCSaphbPaEAQFxEiBHb/pBrRFVOisP8LNQFWWM8DRsYTkyASaNoubbi6q+D6JhZYGkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhodmZhMmR2bTJkYXAAYXT2YXbYKlglAAFxEiAWgI6JK8tfewevuVdhYjcZxYfx/fjAnuJLdWfPmaRT5GFs9oQBAXESIIDttoH/AHk9p7CC5Ly59g3joNwaNZp9Fuuf0BY3QPhAomFlgaRha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZHZvMmRhcABhdPZhdtgqWCUAAXESID7y+1vKWSWi9AOsBnQ7swH7JZMPLCGhrQ09/bnJYlagYWz21AEBcRIgJoaJFS/s99ydlvazK8cquMDO+OpMt2TkHX+BtCBg4O6iYWWBpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkd2EyZGFwAGF02CpYJQABcRIgguQtrjcVELLZ2c2BnB8HvJ20R+KSf2lObeuKIEUIJslhdtgqWCUAAXESIKE2XYySvugWUUYaiSWAkPRM3qvJXfF83tK3Z9IU9hreYWzYKlglAAFxEiAX1eg+J35ruRgaZk4/CQR+h13IBYFthoBu58jaKREm6e8CAXESIBfV6D4nfmu5GBpmTj8JBH6HXcgFgW2GgG7nyNopESbpomFlg6Rha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZHZ3MmRhcABhdNgqWCUAAXESIAwMiDw7jjYfmHpacVnz6By3/GcU0ciN39RgzTHsPc7eYXbYKlglAAFxEiBqBgxQA6lxJllR079EevfZdAupGBkvw0yKbK8BnDoCm6Rha0R3NDJkYXAYH2F02CpYJQABcRIgw20yDvNPQmMr0HmzJpSpDZVng973W2LduQGuFZsm2ClhdtgqWCUAAXESIMgxfLFbe1a+Q+WlezmwL4KvVpdjI5/3yp52T13HVFaapGFrQzcyZGFwGCBhdPZhdtgqWCUAAXESIGSy8mnOZ27dwvsv6vOvZ7LVyg904S6/NJG2yTNatx+QYWzYKlglAAFxEiDK8ZL5WAE8TswPAgwTsS96DN9Zcd/5nCJOIOhGeuUfMK8CAXESIMrxkvlYATxOzA8CDBOxL3oM31lx3/mcIk4g6EZ65R8womFlhKRha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZHZzMmRhcABhdPZhdtgqWCUAAXESIHEwvhH544LeUKZQJSOryGw4sddhjIU+Mfgo1ovMp/T/pGFrQ3QyZGFwGCBhdPZhdtgqWCUAAXESIGCxwohQUIsmHppPtjJaTlII/qT16v3rb7gIoLqInfSYpGFrQ3UyZGFwGCBhdPZhdtgqWCUAAXESIDYnzW2jR5ZrW0937NLiM0eKS87698UdyUZuyIHJM5B7pGFrQ3YyZGFwGCBhdPZhdtgqWCUAAXESIDg0YglLGgWkUp4WHoSpNwelw87W9awVbPzDP0QF7ZKHYWz26QIBcRIgDAyIPDuONh+YelpxWfPoHLf8ZxTRyI3f1GDNMew9zt6iYWWFpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkdngyZGFwAGF09mF22CpYJQABcRIguKGknDMtdA7jJfjRwj1lW14dYNOzHryStUyYC/IaFLikYWtDeTJkYXAYIGF09mF22CpYJQABcRIgaOSuQL6XXY64K+IbjPjYBUliPTES6uAes4nE0Y6Yy1GkYWtDejJkYXAYIGF09mF22CpYJQABcRIg29AXs7My2asUhGQzaCkTcYWeA7NYD5MreNuX9zBJQKikYWtEdzIyZGFwGB9hdPZhdtgqWCUAAXESIF07WmS6/IowaZbl3evdMbPH/pefZS21a458zHg2czY8pGFrQzMyZGFwGCBhdPZhdtgqWCUAAXESIPM2xWsSwD6BlwFYA0+vOeEVA958TTnAwnVsBPx8M+bWYWz2vQEBcRIgw20yDvNPQmMr0HmzJpSpDZVng973W2LduQGuFZsm2CmiYWWCpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkdzUyZGFwAGF09mF22CpYJQABcRIgc5zuv7jNLf7Utzka2o8Ng2kbhcm7G8ZPp4XJ/5oGoyekYWtDNjJkYXAYIGF09mF22CpYJQABcRIgBCp/s3WrGcaMQRvgD2kKnnUNuv2L7Q50nMVQBdTZ9eBhbPbUAQFxEiCC5C2uNxUQstnZzYGcHwe8nbRH4pJ/aU5t64ogRQgmyaJhZYGkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhodmZhMmR3ZDJkYXAAYXTYKlglAAFxEiA93EfEOST8wZzYwz31xHl6TVvGW19dCr1JLXkqsxy2XGF22CpYJQABcRIglesbeYjeQmjVReI84ClHKePHeXnWul7GXyHeYwCcRzdhbNgqWCUAAXESIECS//zsXGDg1mb6k0Kp5c4/Q9ccCJt01FoXRQq5jjQLvQEBcRIgQJL//OxcYODWZvqTQqnlzj9D1xwIm3TUWhdFCrmONAuiYWWCpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkd2IyZGFwAGF09mF22CpYJQABcRIgpwEFeRzmW5ezmwzu8JjWsj/IjinaIgS001U31YasgeikYWtDYzJkYXAYIGF09mF22CpYJQABcRIgBLyDKJtPb6i3CysBLZlbJHEii+yvF8jsv4ESXu3GPSlhbPbqAgFxEiA93EfEOST8wZzYwz31xHl6TVvGW19dCr1JLXkqsxy2XKJhZYWkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhodmZhMmR3ZTJkYXAAYXT2YXbYKlglAAFxEiC+L1AQC/fENm9Kush3R92iPjr6qPVp/DBcuqYokJO9A6Rha0NmMmRhcBggYXT2YXbYKlglAAFxEiDtk8I5Nx+VnOXFg115DXGMVrglP7tdxLNpbScLaxyISaRha0NnMmRhcBggYXT2YXbYKlglAAFxEiBc5tJnif6mOfGo934i85fxuTeukvseqn1f0aT5YpMt46Rha0Vldm8yZGFwGB5hdPZhdtgqWCUAAXESIHN94UiMQkLnRAa4OLvm6wOVjpv/+ynxzSL+IEJ4O9JvpGFrQ3AyZGFwGCBhdPZhdtgqWCUAAXESIA7hd+4njIooRrONJyyaGdsDMvkevc/JO2g8+VncHMxnYWz21gEBcRIg64i6Wm2DQiUsO8ZqArDViBvs99ZMGato3Xd7YONVJkCiYWWBpGFrWCVhcHAuYnNreS5ncmFwaC5saXN0aXRlbS8zbGFxd3M2aHF1azIyYXAAYXTYKlglAAFxEiCzxXVQTDNZo1RUi6cm1K8rVMQ0qQX/zF69CHuq1ukZUGF22CpYJQABcRIgg7k/spSWMNWtVl4pJwn8hlo1EkFRFk3af54QDoFoqkxhbNgqWCUAAXESIK6XhQp0Z1Bsyi8B/26xVelPkp1o/SUrxbwVVL6nFH9G/AIBcRIgrpeFCnRnUGzKLwH/brFV6U+SnWj9JSvFvBVUvqcUf0aiYWWDpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJldnMyZGFwAGF09mF22CpYJQABcRIgEuEFGl293Tpd4Xpf10GCVQMlhrpmxFRv6s0JtMoggeKkYWtDdDJkYXAYIGF02CpYJQABcRIgJ8Ei1Th+RlP6/SulsCpFvY5fpEqIVRQ5SMB0rKrUi1JhdtgqWCUAAXESILXP50rvgJ0nmQUu/7vERJTqBUTjLNrlRsyWqkrjWpFapGFrUmxpc3QvM2xhcXdzNmUyYWkydWFwD2F02CpYJQABcRIgjfdOWsoWPFwM5YsYLdIYKupfaKZiYs7WSgCB36vkl/RhdtgqWCUAAXESIG1EK0I8ZT+0hg5XgFogqwDm/WEHzfeZHs8AbmPqqFXJYWzYKlglAAFxEiCNavG1x4QLL0a/TVeDmr3wMNQm6G3de5zGjR73GX9rCIQBAXESII1q8bXHhAsvRr9NV4OavfAw1Cbobd17nMaNHvcZf2sIomFlgaRha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZXZyMmRhcABhdPZhdtgqWCUAAXESIJRfNfUHYHJ0C0a3wxu4D9OTSBoCaR+vUY8+FPMeQgUYYWz2hAEBcRIgJ8Ei1Th+RlP6/SulsCpFvY5fpEqIVRQ5SMB0rKrUi1KiYWWBpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJldnUyZGFwAGF09mF22CpYJQABcRIgJeSskVxWVBuodAVXOc4fFQx7lkCNRlNj5eiYpymYCgNhbPaPAQFxEiAYBPwuP7sUBINAfeUKFHuC7yJYDZexMYGnw/fviDMoWqNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6YWMybjZvb2EzNjY0eHVid2gzdGtqdnhwaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgLGJ77hDuLN9DiugWUd1zLbn+k4vBGxzlnUYn+h8OQlijZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOm1uNHVjNjIyM2J5MjZscGk0a2V1eW1idmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESID/MpqzAvUwix2r/MOi626k0BfzsupdtJUjc9v6bmDANo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpzcGZyeG1sNWFtcW9mYzdncGpwZ3I0YWhpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiDWdP5tBPqSFPWqfEVzFS+tzFAjZ7xiXQbQEuqsxjHqu6NlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6dG1kcXc1eGZuM3ZhemlwaHBoaGs3cGFsaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgJNoIf2FFjPrbocRc80TC2J81DZ8cr6e3JnLXlz8hL/qjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmNkbW9lbW0zc3B2eW5land0NzU0bDdyemljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIMWOmgzETdyJo3PbgN760UJyPhMB2if61dfrU73nINpYo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpleWJhcHF0bXB0Zm4zc2R0cXJ4cXVtM3FpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiC8SaZXrFI+c6oTp7h4nTND6YKav/t9B/lgnVOhAMJJqqNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6MnNtemdhaHQyNnY1aDVpczJleXhheHB3aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgPQ19l/VvTZlDyzTA2JvYqoJI9kEeM3bPVBEmJynmR6KjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmE3YWJmdmlocmNhNGZzaGxmZnp6dDZ3dGljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIBaAjokry197B6+5V2FiNxnFh/H9+MCe4kt1Z8+ZpFPko2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpyYmFsMjRiMnl2Nnp1eXJ6MjVoNnV0Z29pY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiBU/XL+ykyFDqaEot2Eh5o5T6916TZkvRZjiHTFzBPXI6NlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6aW96djZqenF4emxtbXJocjd3bmR5eWpjaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgPvL7W8pZJaL0A6wGdDuzAfslkw8sIaGtDT39ucliVqCjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmRwZnFlZTRsNHR4NG5lNmJmdmc3M2dzMmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIGecj2nomg9oBsl0rV7yXPqqP9tTgDpg8Q1zMvzPEuTco2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzp5cTdwczZhc2NzNDdtZXJxNTJ3cDNybnNpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiC6huSERRicaWq5rtMR1QlZiaSL/SdI6Lo5veVzhHtWSaNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6eG9mYXRyeXM3NnFkdTJ6dmxhYmVyeHpiaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgL5JLYymSID11odTUeZP43lr7SJIyLo4d1a+zmb+9VSWjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOjRycGE1YzNocWNucDd3ZHhicHNud2czeWljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIHEwvhH544LeUKZQJSOryGw4sddhjIU+Mfgo1ovMp/T/o2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpxNTU3aWJocW9jb2g0MmRheGFmMnNqc2JpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiBgscKIUFCLJh6aT7YyWk5SCP6k9er962+4CKC6iJ30mKNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6ZG1rdDR0cGhqM2dheHVscmtla2Q0ZWx1aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgNifNbaNHlmtbT3fs0uIzR4pLzvr3xR3JRm7IgckzkHujZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmpnb2c3M2lnNHk0NzdrNm5zMml1ZGp1ZmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIDg0YglLGgWkUp4WHoSpNwelw87W9awVbPzDP0QF7ZKHo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzozamtkb3lmams1NGpiZHBwd29qemp5aHlpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiBqBgxQA6lxJllR079EevfZdAupGBkvw0yKbK8BnDoCm6NlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6bHA3YXhobHd0bDVmbDJwcmVsbmdnbjU3aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIguKGknDMtdA7jJfjRwj1lW14dYNOzHryStUyYC/IaFLijZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOm5nbHc2YXczNWRsYWt2a240dGFmNjRnemljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIGjkrkC+l12OuCviG4z42AVJYj0xEurgHrOJxNGOmMtRo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpsaG1yeG5xZnVseTVmc254aWpwcTdleXppY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiDb0BezszLZqxSEZDNoKRNxhZ4Ds1gPkyt425f3MElAqKNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6NGhudjNveGkyN2NvY2p2dWZzcGJ2NHl0aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgXTtaZLr8ijBpluXd690xs8f+l59lLbVrjnzMeDZzNjyjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOnpva25qcjV6b2pueHR4eGM3cmlneWM1eWljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIPM2xWsSwD6BlwFYA0+vOeEVA958TTnAwnVsBPx8M+bWo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpzdjVnaGoydXVsaTZlc2N3a3ZtYnV1a3hpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiDIMXyxW3tWvkPlpXs5sC+Cr1aXYyOf98qedk9dx1RWmqNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6NzNrNjYyaTd0Y2l1b2x6YW11bmh2NTdsaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgc5zuv7jNLf7Utzka2o8Ng2kbhcm7G8ZPp4XJ/5oGoyejZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOnZpaWluNG1hdWdtMmZ5a3pha2N5N2tjMmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIAQqf7N1qxnGjEEb4A9pCp51Dbr9i+0OdJzFUAXU2fXgo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpueWVvY2FwdHRyY2FpenJxd3FhcjZyeGlpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiBksvJpzmdu3cL7L+rzr2ey1coPdOEuvzSRtskzWrcfkKNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6Nm1tc2xhNDJwdm1temZuaXJoNTVtc3BiaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgoTZdjJK+6BZRRhqJJYCQ9Ezeq8ld8Xze0rdn0hT2Gt6jZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOnhmbGxiYjY0amZpaDJ4cnRsdG96bWQybmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIKcBBXkc5luXs5sM7vCY1rI/yI4p2iIEtNNVN9WGrIHoo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzo0Zm9yYWZvbnhld3YyYzNrempvZnZkNmdpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiAEvIMom09vqLcLKwEtmVskcSKL7K8XyOy/gRJe7cY9KaNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6enl2cWxibGphMzdrM2w3N2prYnpqY2d5aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIglesbeYjeQmjVReI84ClHKePHeXnWul7GXyHeYwCcRzejZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmVxZmEyb2hwNDJmcGNqbDU3bTN6c3d0b2ljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIL4vUBAL98Q2b0q6yHdH3aI+Ovqo9Wn8MFy6piiQk70Do2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzp2M3dhZWZzc2tmM3ByM3VjY3o0eHc0ejRpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiDtk8I5Nx+VnOXFg115DXGMVrglP7tdxLNpbScLaxyISaNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6YmNycXh3MjJ6NzJlYXdsdmQyZDRubnY3aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgXObSZ4n+pjnxqPd+IvOX8bk3rpL7Hqp9X9Gk+WKTLeOjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOm81Mmlpd3dhZG9lb3p2eGEyNnhhNXo2Z2ljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIHN94UiMQkLnRAa4OLvm6wOVjpv/+ynxzSL+IEJ4O9Jvo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzozaWVrZnpneTZjM2prY21pb3RubnpobGVpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiAO4XfuJ4yKKEazjScsmhnbAzL5Hr3PyTtoPPlZ3BzMZ6NlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6dXl1a2ZkM3l1dnF2NDNobXY1YnAzaWZyaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgO60lExR61144kov6YcxiRedTUWE2r4J704OTicFKArKjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmRjcXR0dno3Y3RvaGw2aWVsYWE1aXE0NWljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIJRfNfUHYHJ0C0a3wxu4D9OTSBoCaR+vUY8+FPMeQgUYo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpja3p2Mm12dnd1amxtamczaWc2bng2cjRpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiAS4QUaXb3dOl3hel/XQYJVAyWGumbEVG/qzQm0yiCB4qNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6cHR2cTNpNzNpNGdza2xpc295Yzdjb3NwaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgtc/nSu+AnSeZBS7/u8RElOoFROMs2uVGzJaqSuNakVqjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOjZpb3lyZWdkN2FqZnlxc2s3YXN1cDZlcWljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESICXkrJFcVlQbqHQFVznOHxUMe5ZAjUZTY+XomKcpmAoDo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzp4cHJobnAyeHhzczMyYjVscGNuem9xdGppY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlrgAQFxEiCrKtEjLw3+r4t8bEHAnvcRsQx6WLaqm0ZwWsPa/njGO6ZjZGlkeCBkaWQ6cGxjOnZpeGl6Z2p1d2VmNW92bWl4cXhhbGZ1cGNyZXZtM2xoaHZmYW5tb3MyMmNzaWdYQLnvioSbs2Xqh0UVTxeh+6ZJ7czVQZoVoYfj24Y6iQ6ASSJ+sz3e9N/d8OY94vw4t0VnBg/aK8n2o0eSTmPmq09kZGF0YdgqWCUAAXESIHse2ms5gXosdeh4XqvsKvaowb2Y9v8zVOZR6ZlkFK6IZHByZXb2Z3ZlcnNpb24D" + }, + "commit": { + "$link": "bafyreiflflisglyn72xyw7dmihaj55yrweghuwfwvknum4c2ypnp46gghm" + }, + "ops": [ + { + "action": "create", + "cid": { + "$link": "bafyreiayat6c4p53cqcigqd54ufbi64c54rfqdmxweyydj6d67xyqmzili" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dve2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreibmmj564ehoftpuhcxiczi524znxh7jhc6bdmoolhkge75b6dscla" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvf2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreib7zstkzqf5jqrmo2x7gdulvw5jgqc7z3f2s5wsksg4637jxgbqbu" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvg2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreigwot7g2bh2sikplkt4ivzrkl5nzricgz54mjoqnuas5kwmmmpkxm" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvh2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreibe3ieh6ykfrt5nxioeltzujqwyt42q3hy4v6t3ojts26lt6ijp7i" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvi2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreigfr2nazrcn3se2g463qdppvukcoi7bgao2e75nlv7lko66oig2la" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvj2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreif4jgtfplcshzz2ue5hxb4j2m2d5gbjvp73pud7sye5koqqbqsjvi" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvk2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreib5bv6zp5lpjwmuhszuydmjxwfkqjepmqi6gn3m6vareytstzshui" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvl2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreiawqchisk6ll55qpl5zk5qwenyzywd7d7pyycpoes3vm7hztjct4q" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvm2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreicu7vzp5ssmquhknbfc3wcipgrzj6xxl2jwms6rmy4iotc4ye6xem" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvn2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreib66l5vxsszewrpia5maz2dxmyb7mszgdzmegq22dj57w44syswua" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvo2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreidhtshwt2e2b5uansluvvppexh2vi75wu4ahjqpcdltgl6m6exe3q" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvp2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreif2q3siiriytruwvono2mi5kckzrgsix7jhjduluon54vzyi62wje" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvq2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreibpsjfwgkmsea6xliou2r4zh6g6ll5urersf2hb3vnpwom37pkveu" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvr2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreidrgc7bd6pdqlpfbjsqeur2xsdmhcy5oymmqu7dd6bi22f4zj7u74" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvs2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreidawhbiqucqrmtb5gspwyzfutssbd7kj5pk7xvw7oaiuc5irhputa" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvt2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreibwe7gw3i2hszvvwt3x5tjoem2hrjf456xxyuo4srtozca4sm4qpm" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvu2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreibygrrassy2awsffhqwd2cksnyhuxb45vxvvqkwz7gdh5cal3msq4" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvv2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreidkaygfaa5joetfsuotx5chv56zoqf2sgazf7buzctmv4azyoqctm" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvw2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreifyugsjymznoqhogjpy2hbd2zk3lyowbu5td26jfnkmtaf7egquxa" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvx2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreidi4sxebpuxlwhlqk7cdogprwafjfrd2mis5lqb5m4jytiy5gglke" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvy2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreig32al3hmzs3gvrjbdegnucse3rqwpahm2yb6jsw6g3s73taskava" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dvz2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreic5hnngjox4riygtfxf3xv52mnty77jph3ffw2wxdt4zr4dm4zwhq" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dw22d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreihtg3cwwewah2azoakyanh26opbcub547cnhhame5lmat6hym7g2y" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dw32d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreigigf6lcw33k27ehznfpm43al4cv5ljoyzdt734vhtwj5o4ovcwti" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dw42d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreidtttxl7ognfx7njnzzdlni6dmdnenylsn3dpde7j4fzh7zubvde4" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dw52d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreiaefj73g5nldhdiyqi34ahwscu6oug3v7ml5uhhjhgfkac5jwpv4a" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dw62d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreidewlzgttthn3o4f6zp5lz26z5s2xfa65hbf27tjenwzezvvny7sa" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dw72d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreifbgzoyzev65alfcrq2resybehujtpkxsk56f6n5uvxm7jbj5q23y" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dwa2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreifhaecxshhglol3hgym53yjrvvsh7ei4ko2eiclju2vg7kynleb5a" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dwb2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreiaexsbsrg2pn6uloczlaewzswzeoerix3fpc7eozp4bcjpo3rr5fe" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dwc2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreiev5mnxtcg6ijunkrpchtqcsrzj4pdxs6owxjpmmxzb3zrqbhchg4" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dwd2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreif6f5ibac7xyq3g6sv2zb3upxnchy5pvkhvnh6daxf2uyujbe55am" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dwe2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreihnspbdsny7swoolrmdlv4q24mmk24ckp53lxclg2lne4fwwheije" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dwf2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreic443jgpcp6uy47dkhxpyrphf7rxe325ex3d2vh2x6rut4wfezn4m" + }, + "path": "app.bsky.graph.follow/3lhhvfa2dwg2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreidtpxqurdcciltuibvyhc56n2ydswhjx773fhy42ix6ebbhqo6sn4" + }, + "path": "app.bsky.graph.follow/3lhhvfa2evo2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreiao4f364j4mriuenm4ne4wjugo3amzpshv5z7etw2b47fm5yhgmm4" + }, + "path": "app.bsky.graph.follow/3lhhvfa2evp2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreib3vusrgfd225pdreul7jq4yysf45jvcyjwv6bhxu4dsoe4csqcwi" + }, + "path": "app.bsky.graph.follow/3lhhvfa2evq2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreieul427kb3aoj2awrvxymn3qd6tsnebuatjd6xvddz6ctzr4qqfda" + }, + "path": "app.bsky.graph.follow/3lhhvfa2evr2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreias4ecruxn53u5f3yl2l7ludasvamsynotgyrkg72wnbg2muieb4i" + }, + "path": "app.bsky.graph.follow/3lhhvfa2evs2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreifvz7tuv34atutzsbjo7654ireu5icujyzm3lsuntewvjfogwurli" + }, + "path": "app.bsky.graph.follow/3lhhvfa2evt2d" + }, + { + "action": "create", + "cid": { + "$link": "bafyreibf4swjcxcwkqn2q5afk4444hyvbr5zmqenizjwhzpitctstgakam" + }, + "path": "app.bsky.graph.follow/3lhhvfa2evu2d" + } + ], + "prev": null, + "rebase": false, + "repo": "did:plc:vixizgjuwef5ovmixqxalfup", + "rev": "3lhhvfanmos22", + "seq": 4621332152, + "since": "3lhhvfafwmq2r", + "time": "2025-02-06T01:05:27.125Z", + "tooBig": false +} \ No newline at end of file diff --git a/atproto/repo/testdata/firehose_commit_4623075231.json b/atproto/repo/testdata/firehose_commit_4623075231.json new file mode 100644 index 000000000..379713238 --- /dev/null +++ b/atproto/repo/testdata/firehose_commit_4623075231.json @@ -0,0 +1,26 @@ +{ + "blobs": null, + "blocks": { + "$bytes": "OqJlcm9vdHOB2CpYJQABcRIguGgY6Gyzx0qGvu6xnMOvrseCNY9Vnayk3st3mURyKvZndmVyc2lvbgGkAQFxEiBZNQKb1PlJdfWo2pm805YjDrwKuliSKPqEOLCxNUrJlKJhZYGkYWtYG2FwcC5ic2t5LmFjdG9yLnByb2ZpbGUvc2VsZmFwAGF02CpYJQABcRIgRSEckyU/NMy/Gdrs1dSwVHYml/TnosIx/NbKvmnC21BhdtgqWCUAAXESIGT2zjtHHx/OeWyYjkAdNmJwRtOlg6ws+SBm8KaF+dKcYWz2zwEBcRIgZPbOO0cfH855bJiOQB02YnBG06WDrCz5IGbwpoX50pykZSR0eXBldmFwcC5ic2t5LmFjdG9yLnByb2ZpbGVmYXZhdGFypGNyZWbYKlglAAFVEiA2ONI9RhyxWEPzAjacznhfqSPC60Y/75u0/HVdqVgNXGRzaXplGTxUZSR0eXBlZGJsb2JobWltZVR5cGVqaW1hZ2UvanBlZ2ljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6NDM6MjEuMjk5WmtkaXNwbGF5TmFtZWDgAQFxEiC4aBjobLPHSoa+7rGcw6+ux4I1j1WdrKTey3eZRHIq9qZjZGlkeCBkaWQ6cGxjOnN4dG9hczJzeGp2NmdoMmJpYmRnM2c0YmNyZXZtM2xoaHhpenU1d3IyZGNzaWdYQHBHEeGz9EBPjzt9aQpFHE0iOFUpCiL1SYSe5tpTO2tlHF26TAoeSWhvQnvjd/aDVfVKDnQ/dLABLB7MYnweNexkZGF0YdgqWCUAAXESIFk1ApvU+Ul19ajambzTliMOvAq6WJIo+oQ4sLE1SsmUZHByZXb2Z3ZlcnNpb24D" + }, + "commit": { + "$link": "bafyreifynamoq3fty5finpxowgomhl5oy6bdld2vtwwkjxwlo6mui4rk6y" + }, + "ops": [ + { + "action": "create", + "cid": { + "$link": "bafyreide63hdwry7d7hhs3eyrzab2ntcobdnhjmdvqwpsidg6ctil6ostq" + }, + "path": "app.bsky.actor.profile/self" + } + ], + "prev": null, + "rebase": false, + "repo": "did:plc:sxtoas2sxjv6gh2bibdg3g4b", + "rev": "3lhhxizu5wr2d", + "seq": 4623075231, + "since": "3lhhxiz32k52v", + "time": "2025-02-06T01:43:22.273Z", + "tooBig": false +} \ No newline at end of file diff --git a/atproto/repo/tiny_blockstore.go b/atproto/repo/tiny_blockstore.go new file mode 100644 index 000000000..c332e24dc --- /dev/null +++ b/atproto/repo/tiny_blockstore.go @@ -0,0 +1,33 @@ +package repo + +import ( + "context" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" +) + +type TinyBlockstore struct { + blocks map[string]blocks.Block +} + +func NewTinyBlockstore() *TinyBlockstore { + return &TinyBlockstore{blocks: make(map[string]blocks.Block, 20)} +} + +func (tb *TinyBlockstore) Put(_ context.Context, block blocks.Block) error { + ncid := block.Cid() + key := ncid.KeyString() + tb.blocks[key] = block + return nil +} + +func (tb *TinyBlockstore) Get(_ context.Context, ncid cid.Cid) (blocks.Block, error) { + key := ncid.KeyString() + block, found := tb.blocks[key] + if found { + return block, nil + } + return nil, &ipld.ErrNotFound{Cid: ncid} +} diff --git a/atproto/syntax/atidentifier.go b/atproto/syntax/atidentifier.go new file mode 100644 index 000000000..14f03a31a --- /dev/null +++ b/atproto/syntax/atidentifier.go @@ -0,0 +1,86 @@ +package syntax + +import ( + "errors" + "strings" +) + +type AtIdentifier string + +func ParseAtIdentifier(raw string) (AtIdentifier, error) { + if raw == "" { + return "", errors.New("expected AT account identifier, got empty string") + } + if strings.HasPrefix(raw, "did:") { + did, err := ParseDID(raw) + if err != nil { + return "", err + } + return AtIdentifier(did), nil + } + handle, err := ParseHandle(raw) + if err != nil { + return "", err + } + return AtIdentifier(handle), nil +} + +func (n AtIdentifier) IsHandle() bool { + return n != "" && !n.IsDID() +} + +func (n AtIdentifier) AsHandle() (Handle, error) { + if n.IsHandle() { + return Handle(n), nil + } + return "", errors.New("AT Identifier is not a Handle") +} + +func (n AtIdentifier) Handle() Handle { + if n.IsHandle() { + return Handle(n) + } + return "" +} + +func (n AtIdentifier) IsDID() bool { + return strings.HasPrefix(n.String(), "did:") +} + +func (n AtIdentifier) AsDID() (DID, error) { + if n.IsDID() { + return DID(n), nil + } + return "", errors.New("AT Identifier is not a DID") +} + +func (n AtIdentifier) DID() DID { + if n.IsDID() { + return DID(n) + } + return "" +} + +func (n AtIdentifier) Normalize() AtIdentifier { + if n.IsHandle() { + return Handle(n).Normalize().AtIdentifier() + } + return n +} + +func (n AtIdentifier) String() string { + return string(n) +} + +func (a AtIdentifier) MarshalText() ([]byte, error) { + return []byte(a.String()), nil +} + +func (a *AtIdentifier) UnmarshalText(text []byte) error { + atid, err := ParseAtIdentifier(string(text)) + if err != nil { + return err + } + *a = atid + return nil +} diff --git a/atproto/syntax/atidentifier_test.go b/atproto/syntax/atidentifier_test.go new file mode 100644 index 000000000..5ac6e4d0e --- /dev/null +++ b/atproto/syntax/atidentifier_test.go @@ -0,0 +1,85 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropAtIdentifiersValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/atidentifier_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseAtIdentifier(line) + if err != nil { + fmt.Println("FAILED, GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropAtIdentifiersInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/atidentifier_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseAtIdentifier(line) + if err == nil { + fmt.Println("FAILED, BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestDowncase(t *testing.T) { + assert := assert.New(t) + + aidh, err := ParseAtIdentifier("example.com") + assert.NoError(err) + assert.True(aidh.IsHandle()) + assert.False(aidh.IsDID()) + _, err = aidh.AsHandle() + assert.NoError(err) + _, err = aidh.AsDID() + assert.Error(err) + + aidd, err := ParseAtIdentifier("did:web:example.com") + assert.NoError(err) + assert.False(aidd.IsHandle()) + assert.True(aidd.IsDID()) + _, err = aidd.AsHandle() + assert.Error(err) + _, err = aidd.AsDID() + assert.NoError(err) +} + +func TestEmpty(t *testing.T) { + assert := assert.New(t) + + atid := AtIdentifier("") + + assert.False(atid.IsHandle()) + assert.False(atid.IsDID()) + assert.Equal(atid, atid.Normalize()) + atid.AsHandle() + atid.AsDID() + assert.Empty(atid.String()) +} diff --git a/atproto/syntax/aturi.go b/atproto/syntax/aturi.go new file mode 100644 index 000000000..758560014 --- /dev/null +++ b/atproto/syntax/aturi.go @@ -0,0 +1,135 @@ +package syntax + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +var aturiRegex = regexp.MustCompile(`^at:\/\/(?P[a-zA-Z0-9._:%-]+)(\/(?P[a-zA-Z0-9-.]+)(\/(?P[a-zA-Z0-9_~.:-]{1,512}))?)?$`) + +// String type which represents a syntaxtually valid AT URI, as would pass Lexicon syntax validation for the 'at-uri' field (no query or fragment parts) +// +// Always use [ParseATURI] instead of wrapping strings directly, especially when working with input. +// +// Syntax specification: https://atproto.com/specs/at-uri-scheme +type ATURI string + +func ParseATURI(raw string) (ATURI, error) { + if len(raw) > 8192 { + return "", errors.New("ATURI is too long (8192 chars max)") + } + parts := aturiRegex.FindStringSubmatch(raw) + if parts == nil || len(parts) < 2 || parts[0] == "" { + return "", errors.New("AT-URI syntax didn't validate via regex") + } + // verify authority as either a DID or NSID + _, err := ParseAtIdentifier(parts[1]) + if err != nil { + return "", fmt.Errorf("AT-URI authority section neither a DID nor Handle: %s", parts[1]) + } + if len(parts) >= 4 && parts[3] != "" { + _, err := ParseNSID(parts[3]) + if err != nil { + return "", fmt.Errorf("AT-URI first path segment not an NSID: %s", parts[3]) + } + } + if len(parts) >= 6 && parts[5] != "" { + _, err := ParseRecordKey(parts[5]) + if err != nil { + return "", fmt.Errorf("AT-URI second path segment not a RecordKey: %s", parts[5]) + } + } + return ATURI(raw), nil +} + +// Every valid ATURI has a valid AtIdentifier in the authority position. +// +// If this ATURI is malformed, returns empty +func (n ATURI) Authority() AtIdentifier { + parts := strings.SplitN(string(n), "/", 4) + if len(parts) < 3 { + // something has gone wrong (would not validate) + return "" + } + atid, err := ParseAtIdentifier(parts[2]) + if err != nil { + return "" + } + return atid +} + +// Returns path segment, without leading slash, as would be used in an atproto repository key. Or empty string if there is no path. +func (n ATURI) Path() string { + parts := strings.SplitN(string(n), "/", 5) + if len(parts) < 4 { + // something has gone wrong (would not validate) + return "" + } + if len(parts) == 4 { + return parts[3] + } + return parts[3] + "/" + parts[4] +} + +// Returns a valid NSID if there is one in the appropriate part of the path, otherwise empty. +func (n ATURI) Collection() NSID { + parts := strings.SplitN(string(n), "/", 5) + if len(parts) < 4 { + // something has gone wrong (would not validate) + return NSID("") + } + nsid, err := ParseNSID(parts[3]) + if err != nil { + return NSID("") + } + return nsid +} + +func (n ATURI) RecordKey() RecordKey { + parts := strings.SplitN(string(n), "/", 6) + if len(parts) < 5 { + // something has gone wrong (would not validate) + return RecordKey("") + } + rkey, err := ParseRecordKey(parts[4]) + if err != nil { + return RecordKey("") + } + return rkey +} + +func (n ATURI) Normalize() ATURI { + auth := n.Authority() + if auth == "" { + // invalid AT-URI; return the current value (!) + return n + } + coll := n.Collection() + if coll == NSID("") { + return ATURI("at://" + auth.Normalize().String()) + } + rkey := n.RecordKey() + if rkey == RecordKey("") { + return ATURI("at://" + auth.Normalize().String() + "/" + coll.String()) + } + return ATURI("at://" + auth.Normalize().String() + "/" + coll.Normalize().String() + "/" + rkey.String()) +} + +func (n ATURI) String() string { + return string(n) +} + +func (a ATURI) MarshalText() ([]byte, error) { + return []byte(a.String()), nil +} + +func (a *ATURI) UnmarshalText(text []byte) error { + aturi, err := ParseATURI(string(text)) + if err != nil { + return err + } + *a = aturi + return nil +} diff --git a/atproto/syntax/aturi_test.go b/atproto/syntax/aturi_test.go new file mode 100644 index 000000000..4ba278300 --- /dev/null +++ b/atproto/syntax/aturi_test.go @@ -0,0 +1,122 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropATURIsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/aturi_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + aturi, err := ParseATURI(line) + if err != nil { + fmt.Println("FAILED, GOOD: " + line) + } + assert.NoError(err) + + // check that Path() is working + col := aturi.Collection() + rkey := aturi.RecordKey() + if rkey != "" { + assert.Equal(col.String()+"/"+rkey.String(), aturi.Path()) + } else if col != "" { + assert.Equal(col.String(), aturi.Path()) + } + } + assert.NoError(scanner.Err()) +} + +func TestInteropATURIsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/aturi_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseATURI(line) + if err == nil { + fmt.Println("FAILED, BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestATURIParts(t *testing.T) { + assert := assert.New(t) + + testVec := [][]string{ + {"at://did:abc:123/io.nsid.someFunc/record-key", "did:abc:123", "io.nsid.someFunc", "record-key"}, + {"at://e.com", "e.com", "", ""}, + } + + for _, parts := range testVec { + uri, err := ParseATURI(parts[0]) + assert.NoError(err) + auth := uri.Authority() + assert.Equal(parts[1], auth.String()) + col := uri.Collection() + assert.Equal(parts[2], col.String()) + rkey := uri.RecordKey() + assert.Equal(parts[3], rkey.String()) + } +} + +func TestATURIPath(t *testing.T) { + assert := assert.New(t) + + uri1, err := ParseATURI("at://did:abc:123/io.nsid.someFunc/record-key") + assert.NoError(err) + assert.Equal("io.nsid.someFunc/record-key", uri1.Path()) + + uri2, err := ParseATURI("at://did:abc:123/io.nsid.someFunc") + assert.NoError(err) + assert.Equal("io.nsid.someFunc", uri2.Path()) + + uri3, err := ParseATURI("at://did:abc:123") + assert.NoError(err) + assert.Equal("", uri3.Path()) +} + +func TestATURINormalize(t *testing.T) { + assert := assert.New(t) + + testVec := [][]string{ + {"at://did:abc:123/io.NsId.someFunc/record-KEY", "at://did:abc:123/io.nsid.someFunc/record-KEY"}, + {"at://E.com", "at://e.com"}, + } + + for _, parts := range testVec { + uri, err := ParseATURI(parts[0]) + assert.NoError(err) + assert.Equal(parts[1], uri.Normalize().String()) + } +} + +func TestATURINoPanic(t *testing.T) { + for _, s := range []string{"", ".", "at://", "at:///", "at://e.com", "at://e.com/", "at://e.com//"} { + bad := ATURI(s) + _ = bad.Authority() + _ = bad.Collection() + _ = bad.RecordKey() + _ = bad.Normalize() + _ = bad.String() + _ = bad.Path() + } +} diff --git a/atproto/syntax/cid.go b/atproto/syntax/cid.go new file mode 100644 index 000000000..c6ad616c6 --- /dev/null +++ b/atproto/syntax/cid.go @@ -0,0 +1,53 @@ +package syntax + +import ( + "errors" + "regexp" + "strings" +) + +// Represents a CIDv1 in string format, as would pass Lexicon syntax validation. +// +// You usually want to use the github.com/ipfs/go-cid package and type when working with CIDs ("Links") in atproto. This specific type (syntax.CID) is an informal/incomplete helper specifically for doing fast string verification or pass-through without parsing, re-serialization, or normalization. +// +// Always use [ParseCID] instead of wrapping strings directly, especially when working with network input. +type CID string + +var cidRegex = regexp.MustCompile(`^[a-zA-Z0-9+=]{8,256}$`) + +func ParseCID(raw string) (CID, error) { + if raw == "" { + return "", errors.New("expected CID, got empty string") + } + if len(raw) > 256 { + return "", errors.New("CID is too long (256 chars max)") + } + if len(raw) < 8 { + return "", errors.New("CID is too short (8 chars min)") + } + + if !cidRegex.MatchString(raw) { + return "", errors.New("CID syntax didn't validate via regex") + } + if strings.HasPrefix(raw, "Qmb") { + return "", errors.New("CIDv0 not allowed in this version of atproto") + } + return CID(raw), nil +} + +func (c CID) String() string { + return string(c) +} + +func (c CID) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c *CID) UnmarshalText(text []byte) error { + cid, err := ParseCID(string(text)) + if err != nil { + return err + } + *c = cid + return nil +} diff --git a/atproto/syntax/cid_test.go b/atproto/syntax/cid_test.go new file mode 100644 index 000000000..7da07c6c2 --- /dev/null +++ b/atproto/syntax/cid_test.go @@ -0,0 +1,50 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropCIDsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/cid_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseCID(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropCIDsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/cid_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseCID(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} diff --git a/atproto/syntax/cmd/atp-syntax/main.go b/atproto/syntax/cmd/atp-syntax/main.go new file mode 100644 index 000000000..7289d13fa --- /dev/null +++ b/atproto/syntax/cmd/atp-syntax/main.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "atp-syntax", + Usage: "informal debugging CLI tool for atproto syntax (identifiers)", + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "parse-tid", + Usage: "parse a TID and output timestamp", + ArgsUsage: "", + Action: runParseTID, + }, + &cli.Command{ + Name: "parse-did", + Usage: "parse a DID", + ArgsUsage: "", + Action: runParseDID, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + if err := app.Run(context.Background(), os.Args); err != nil { + slog.Error("command failed", "error", err) + os.Exit(-1) + } +} + +func runParseTID(ctx context.Context, cmd *cli.Command) error { + s := cmd.Args().First() + if s == "" { + return fmt.Errorf("need to provide identifier as an argument") + } + + tid, err := syntax.ParseTID(s) + if err != nil { + return err + } + fmt.Printf("TID: %s\n", tid) + fmt.Printf("Time: %s\n", tid.Time()) + + return nil +} + +func runParseDID(ctx context.Context, cmd *cli.Command) error { + s := cmd.Args().First() + if s == "" { + return fmt.Errorf("need to provide identifier as an argument") + } + + did, err := syntax.ParseDID(s) + if err != nil { + return err + } + fmt.Printf("%s\n", did) + + return nil +} diff --git a/atproto/syntax/datetime.go b/atproto/syntax/datetime.go new file mode 100644 index 000000000..3658c5d67 --- /dev/null +++ b/atproto/syntax/datetime.go @@ -0,0 +1,130 @@ +package syntax + +import ( + "errors" + "fmt" + "regexp" + "strings" + "time" +) + +const ( + // Preferred atproto Datetime string syntax, for use with [time.Format]. + // + // Note that *parsing* syntax is more flexible. + AtprotoDatetimeLayout = "2006-01-02T15:04:05.999Z" +) + +// Represents the a Datetime in string format, as would pass Lexicon syntax validation: the intersection of RFC-3339 and ISO-8601 syntax. +// +// Always use [ParseDatetime] instead of wrapping strings directly, especially when working with network input. +// +// Syntax is specified at: https://atproto.com/specs/lexicon#datetime +type Datetime string + +var datetimeRegex = regexp.MustCompile(`^[0-9]{4}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9](.[0-9]{1,20})?(Z|([+-][0-2][0-9]:[0-5][0-9]))$`) +var astroZero = time.Date(0000, 1, 1, 0, 0, 0, 0, time.UTC) + +func ParseDatetime(raw string) (Datetime, error) { + if raw == "" { + return "", errors.New("expected datetime, got empty string") + } + if len(raw) > 64 { + return "", errors.New("Datetime too long (max 64 chars)") + } + + if !datetimeRegex.MatchString(raw) { + return "", errors.New("Datetime syntax didn't validate via regex") + } + if strings.HasSuffix(raw, "-00:00") { + return "", errors.New("Datetime can't use '-00:00' for UTC timezone, must use '+00:00', per ISO-8601") + } + // ensure that the datetime actually parses using golang time lib + t, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + return "", err + } + // times before astronomical zero are disallowed + if t.Before(astroZero) { + return "", errors.New("Datetime can not be before year zero") + } + return Datetime(raw), nil +} + +// Validates and converts a string to a golang [time.Time] in a single step. +func ParseDatetimeTime(raw string) (time.Time, error) { + d, err := ParseDatetime(raw) + if err != nil { + var zero time.Time + return zero, err + } + return d.Time(), nil +} + +// Similar to ParseDatetime, but more flexible about some parsing. +// +// Note that this may mutate the internal string, so a round-trip will fail. This is intended for working with legacy/broken records, not to be used in an ongoing way. +var hasTimezoneRegex = regexp.MustCompile(`^.*(([+-]\d\d:?\d\d)|[a-zA-Z])$`) + +func ParseDatetimeLenient(raw string) (Datetime, error) { + // fast path: it is a valid overall datetime + valid, err := ParseDatetime(raw) + if nil == err { + return valid, nil + } + + if strings.HasSuffix(raw, "-00:00") { + return ParseDatetime(strings.Replace(raw, "-00:00", "+00:00", 1)) + } + if strings.HasSuffix(raw, "-0000") { + return ParseDatetime(strings.Replace(raw, "-0000", "+00:00", 1)) + } + if strings.HasSuffix(raw, "+0000") { + return ParseDatetime(strings.Replace(raw, "+0000", "+00:00", 1)) + } + + // try adding timezone if it is missing + if !hasTimezoneRegex.MatchString(raw) { + withTZ, err := ParseDatetime(raw + "Z") + if nil == err { + return withTZ, nil + } + } + + return "", fmt.Errorf("Datetime could not be parsed, even leniently: %v", err) +} + +// Parses the Datetime string in to a golang [time.Time]. +// +// This method assumes that [ParseDatetime] was used to create the Datetime, which already verified parsing, and thus that [time.Parse] will always succeed. In the event of an error, zero/nil will be returned. +func (d Datetime) Time() time.Time { + var zero time.Time + ret, err := time.Parse(time.RFC3339Nano, d.String()) + if err != nil { + return zero + } + return ret +} + +// Creates a new valid Datetime string matching the current time, in preferred syntax. +func DatetimeNow() Datetime { + t := time.Now().UTC() + return Datetime(t.Format(AtprotoDatetimeLayout)) +} + +func (d Datetime) String() string { + return string(d) +} + +func (d Datetime) MarshalText() ([]byte, error) { + return []byte(d.String()), nil +} + +func (d *Datetime) UnmarshalText(text []byte) error { + datetime, err := ParseDatetime(string(text)) + if err != nil { + return err + } + *d = datetime + return nil +} diff --git a/atproto/syntax/datetime_test.go b/atproto/syntax/datetime_test.go new file mode 100644 index 000000000..595840a8f --- /dev/null +++ b/atproto/syntax/datetime_test.go @@ -0,0 +1,115 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropDatetimeValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/datetime_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseDatetimeTime(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropDatetimeInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/datetime_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseDatetime(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropDatetimeTimeInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/datetime_parse_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseDatetime(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + _, err = ParseDatetimeTime(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestParseDatetimeLenient(t *testing.T) { + assert := assert.New(t) + + valid := []string{ + "1985-04-12T23:20:50.123Z", + "1985-04-12T23:20:50.123", + "2023-08-27T19:07:00.186173", + "1985-04-12T23:20:50.123-00:00", + "1985-04-12T23:20:50.123+00:00", + "1985-04-12T23:20:50.123+0000", + "1985-04-12T23:20:50.123-0000", + "2023-11-12T11:20:01+0000", + } + for _, s := range valid { + _, err := ParseDatetimeLenient(s) + assert.NoError(err) + if err != nil { + fmt.Println(s) + } + } + + invalid := []string{ + "1985-04-", + "", + "blah", + } + for _, s := range invalid { + _, err := ParseDatetimeLenient(s) + assert.Error(err) + } +} + +func TestDatetimeNow(t *testing.T) { + assert := assert.New(t) + + dt := DatetimeNow() + _, err := ParseDatetimeTime(dt.String()) + assert.NoError(err) +} diff --git a/atproto/syntax/did.go b/atproto/syntax/did.go new file mode 100644 index 000000000..ffc473f31 --- /dev/null +++ b/atproto/syntax/did.go @@ -0,0 +1,93 @@ +package syntax + +import ( + "errors" + "regexp" + "strings" +) + +// Represents a syntaxtually valid DID identifier, as would pass Lexicon syntax validation. +// +// Always use [ParseDID] instead of wrapping strings directly, especially when working with input. +// +// Syntax specification: https://atproto.com/specs/did +type DID string + +var didRegex = regexp.MustCompile(`^did:[a-z]+:[a-zA-Z0-9._:%-]*[a-zA-Z0-9._-]$`) + +func isASCIIAlphaNum(c rune) bool { + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') { + return true + } + return false +} + +func ParseDID(raw string) (DID, error) { + // fast-path for did:plc, avoiding regex + if len(raw) == 32 && strings.HasPrefix(raw, "did:plc:") { + // NOTE: this doesn't really check base32, just broader alphanumberic. might pass invalid PLC DIDs, but they still have overall valid DID syntax + isPlc := true + for _, c := range raw[8:32] { + if !isASCIIAlphaNum(c) { + isPlc = false + break + } + } + if isPlc { + return DID(raw), nil + } + } + if raw == "" { + return "", errors.New("expected DID, got empty string") + } + if len(raw) > 2*1024 { + return "", errors.New("DID is too long (2048 chars max)") + } + if !didRegex.MatchString(raw) { + return "", errors.New("DID syntax didn't validate via regex") + } + return DID(raw), nil +} + +// The "method" part of the DID, between the 'did:' prefix and the final identifier segment, normalized to lower-case. +func (d DID) Method() string { + // syntax guarantees that there are at least 3 parts of split + parts := strings.SplitN(string(d), ":", 3) + if len(parts) < 2 { + // this should be impossible; return empty to avoid out-of-bounds + return "" + } + return strings.ToLower(parts[1]) +} + +// The final "identifier" segment of the DID +func (d DID) Identifier() string { + // syntax guarantees that there are at least 3 parts of split + parts := strings.SplitN(string(d), ":", 3) + if len(parts) < 3 { + // this should be impossible; return empty to avoid out-of-bounds + return "" + } + return parts[2] +} + +func (d DID) AtIdentifier() AtIdentifier { + return AtIdentifier(d) +} + +func (d DID) String() string { + return string(d) +} + +func (d DID) MarshalText() ([]byte, error) { + return []byte(d.String()), nil +} + +func (d *DID) UnmarshalText(text []byte) error { + did, err := ParseDID(string(text)) + if err != nil { + return err + } + *d = did + return nil +} diff --git a/atproto/syntax/did_test.go b/atproto/syntax/did_test.go new file mode 100644 index 000000000..2c085bc86 --- /dev/null +++ b/atproto/syntax/did_test.go @@ -0,0 +1,69 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropDIDsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/did_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseDID(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropDIDsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/did_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseDID(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestDIDParts(t *testing.T) { + assert := assert.New(t) + d, err := ParseDID("did:example:123456789abcDEFghi") + assert.NoError(err) + assert.Equal("example", d.Method()) + assert.Equal("123456789abcDEFghi", d.Identifier()) + assert.Equal(d.String(), d.AtIdentifier().String()) +} + +func TestDIDNoPanic(t *testing.T) { + for _, s := range []string{"", ":", "::"} { + bad := DID(s) + _ = bad.Identifier() + _ = bad.Method() + _ = bad.AtIdentifier() + _ = bad.AtIdentifier().String() + } +} diff --git a/atproto/syntax/doc.go b/atproto/syntax/doc.go new file mode 100644 index 000000000..40dbaa868 --- /dev/null +++ b/atproto/syntax/doc.go @@ -0,0 +1,4 @@ +// Package syntax provides types for identifiers and other string formats. +// +// These are primarily simple string alias types for parsing or verifying protocol-level syntax of identifiers, not routines for things like resolution or verification against application policies. +package syntax diff --git a/atproto/syntax/handle.go b/atproto/syntax/handle.go new file mode 100644 index 000000000..65c32de6a --- /dev/null +++ b/atproto/syntax/handle.go @@ -0,0 +1,88 @@ +package syntax + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +var ( + handleRegex = regexp.MustCompile(`^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$`) + + // special handle string constant indicating that handle resolution failed + HandleInvalid = Handle("handle.invalid") +) + +// String type which represents a syntaxtually valid handle identifier, as would pass Lexicon syntax validation. +// +// Always use [ParseHandle] instead of wrapping strings directly, especially when working with input. +// +// Syntax specification: https://atproto.com/specs/handle +type Handle string + +func ParseHandle(raw string) (Handle, error) { + if raw == "" { + return "", errors.New("expected handle, got empty string") + } + if len(raw) > 253 { + return "", errors.New("handle is too long (253 chars max)") + } + if !handleRegex.MatchString(raw) { + return "", fmt.Errorf("handle syntax didn't validate via regex: %s", raw) + } + return Handle(raw), nil +} + +// Some top-level domains (TLDs) are disallowed for registration across the atproto ecosystem. The *syntax* is valid, but these should never be considered acceptable handles for account registration or linking. +// +// The reserved '.test' TLD is allowed, for testing and development. It is expected that '.test' domain resolution will fail in a real-world network. +func (h Handle) AllowedTLD() bool { + switch h.TLD() { + case "local", + "arpa", + "invalid", + "localhost", + "internal", + "example", + "onion", + "alt": + return false + } + return true +} + +func (h Handle) TLD() string { + parts := strings.Split(string(h.Normalize()), ".") + return parts[len(parts)-1] +} + +// Is this the special "handle.invalid" handle? +func (h Handle) IsInvalidHandle() bool { + return h.Normalize() == HandleInvalid +} + +func (h Handle) Normalize() Handle { + return Handle(strings.ToLower(string(h))) +} + +func (h Handle) AtIdentifier() AtIdentifier { + return AtIdentifier(h) +} + +func (h Handle) String() string { + return string(h) +} + +func (h Handle) MarshalText() ([]byte, error) { + return []byte(h.String()), nil +} + +func (h *Handle) UnmarshalText(text []byte) error { + handle, err := ParseHandle(string(text)) + if err != nil { + return err + } + *h = handle + return nil +} diff --git a/atproto/syntax/handle_test.go b/atproto/syntax/handle_test.go new file mode 100644 index 000000000..3d591de35 --- /dev/null +++ b/atproto/syntax/handle_test.go @@ -0,0 +1,80 @@ +package syntax + +import ( + "bufio" + "encoding" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropHandlesValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/handle_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseHandle(line) + if err != nil { + fmt.Println("FAILED, GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropHandlesInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/handle_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseHandle(line) + if err == nil { + fmt.Println("FAILED, BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestHandleNormalize(t *testing.T) { + assert := assert.New(t) + + handle, err := ParseHandle("JoHn.TeST") + assert.NoError(err) + assert.Equal("john.test", string(handle.Normalize())) + assert.NoError(err) + assert.Equal(handle.String(), handle.AtIdentifier().String()) + + _, err = ParseHandle("JoH!n.TeST") + assert.Error(err) +} + +func TestHandleNoPanic(t *testing.T) { + for _, s := range []string{"", ".", ".."} { + bad := Handle(s) + _ = bad.Normalize() + _ = bad.TLD() + _ = bad.AllowedTLD() + _ = bad.AtIdentifier() + _ = bad.AtIdentifier().String() + } +} +func TestHandleInterfaces(t *testing.T) { + h := Handle("e.com") + var _ encoding.TextMarshaler = h + var _ encoding.TextUnmarshaler = &h +} diff --git a/atproto/syntax/json_test.go b/atproto/syntax/json_test.go new file mode 100644 index 000000000..5af32adfe --- /dev/null +++ b/atproto/syntax/json_test.go @@ -0,0 +1,84 @@ +package syntax + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestJSONEncoding(t *testing.T) { + assert := assert.New(t) + + type AllTogether struct { + Handle Handle `json:"handle"` + Aturi ATURI `json:"aturi"` + Did DID `json:"did"` + Atid AtIdentifier `json:"atid"` + Rkey RecordKey `json:"rkey"` + Col *NSID `json:"col"` // demonstrating a pointer + } + fullJSON := `{ + "handle": "handle.example.com", + "aturi": "at://handle.example.com/not.atproto.thing/abc123", + "did": "did:abc:123", + "atid": "did:abc:123", + "rkey": "abc123", + "col": "not.atproto.thing" + }` + assert.Equal(json.Valid([]byte(fullJSON)), true) + + handle, err := ParseHandle("handle.example.com") + assert.NoError(err) + aturi, err := ParseATURI("at://handle.example.com/not.atproto.thing/abc123") + assert.NoError(err) + did, err := ParseDID("did:abc:123") + assert.NoError(err) + atid, err := ParseAtIdentifier("did:abc:123") + assert.NoError(err) + rkey, err := ParseRecordKey("abc123") + assert.NoError(err) + col, err := ParseNSID("not.atproto.thing") + assert.NoError(err) + + fullStruct := AllTogether{ + Handle: handle, + Aturi: aturi, + Did: did, + Atid: atid, + Rkey: rkey, + Col: &col, + } + + _, err = json.Marshal(fullStruct) + assert.NoError(err) + + var parseStruct AllTogether + err = json.Unmarshal([]byte(fullJSON), &parseStruct) + assert.NoError(err) + assert.Equal(fullStruct, parseStruct) + + badJSON := `{"handle": 12343}` + err = json.Unmarshal([]byte(badJSON), &parseStruct) + assert.Error(err) + + wrongJSON := `{"handle": "asdf"}` + err = json.Unmarshal([]byte(wrongJSON), &parseStruct) + assert.Error(err) + + okJSON := `{"handle": "blah.com"}` + err = json.Unmarshal([]byte(okJSON), &parseStruct) + assert.NoError(err) +} + +func TestJSONHandle(t *testing.T) { + assert := assert.New(t) + + blob := `["atproto.com", "bsky.app"]` + var handleList []Handle + if err := json.Unmarshal([]byte(blob), &handleList); err != nil { + t.Fatal(err) + } + assert.Equal(Handle("atproto.com"), handleList[0]) + assert.Equal(Handle("bsky.app"), handleList[1]) +} diff --git a/atproto/syntax/language.go b/atproto/syntax/language.go new file mode 100644 index 000000000..59a9d3322 --- /dev/null +++ b/atproto/syntax/language.go @@ -0,0 +1,45 @@ +package syntax + +import ( + "errors" + "regexp" +) + +// Represents a Language specifier in string format, as would pass Lexicon syntax validation. +// +// Always use [ParseLanguage] instead of wrapping strings directly, especially when working with network input. +// +// The syntax is BCP-47. This is a partial/naive parsing implementation, designed for fast validation and exact-string passthrough with no normaliztion. For actually working with BCP-47 language specifiers in atproto code bases, we recommend the golang.org/x/text/language package. +type Language string + +var langRegex = regexp.MustCompile(`^(i|[a-z]{2,3})(-[a-zA-Z0-9]+)*$`) + +func ParseLanguage(raw string) (Language, error) { + if raw == "" { + return "", errors.New("expected language code, got empty string") + } + if len(raw) > 128 { + return "", errors.New("Language is too long (128 chars max)") + } + if !langRegex.MatchString(raw) { + return "", errors.New("Language syntax didn't validate via regex") + } + return Language(raw), nil +} + +func (l Language) String() string { + return string(l) +} + +func (l Language) MarshalText() ([]byte, error) { + return []byte(l.String()), nil +} + +func (l *Language) UnmarshalText(text []byte) error { + lang, err := ParseLanguage(string(text)) + if err != nil { + return err + } + *l = lang + return nil +} diff --git a/atproto/syntax/language_test.go b/atproto/syntax/language_test.go new file mode 100644 index 000000000..96556b0ab --- /dev/null +++ b/atproto/syntax/language_test.go @@ -0,0 +1,50 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropLanguagesValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/language_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseLanguage(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropLanguagesInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/language_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseLanguage(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} diff --git a/atproto/syntax/nsid.go b/atproto/syntax/nsid.go new file mode 100644 index 000000000..9284d7eaa --- /dev/null +++ b/atproto/syntax/nsid.go @@ -0,0 +1,78 @@ +package syntax + +import ( + "errors" + "regexp" + "strings" +) + +var nsidRegex = regexp.MustCompile(`^[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+(\.[a-zA-Z]([a-zA-Z0-9]{0,62})?)$`) + +// String type which represents a syntaxtually valid Namespace Identifier (NSID), as would pass Lexicon syntax validation. +// +// Always use [ParseNSID] instead of wrapping strings directly, especially when working with input. +// +// Syntax specification: https://atproto.com/specs/nsid +type NSID string + +func ParseNSID(raw string) (NSID, error) { + if raw == "" { + return "", errors.New("expected NSID, got empty string") + } + if len(raw) > 317 { + return "", errors.New("NSID is too long (317 chars max)") + } + if !nsidRegex.MatchString(raw) { + return "", errors.New("NSID syntax didn't validate via regex") + } + return NSID(raw), nil +} + +// Authority domain name, in regular DNS order, not reversed order, normalized to lower-case. +func (n NSID) Authority() string { + parts := strings.Split(string(n), ".") + if len(parts) < 2 { + // something has gone wrong (would not validate); return empty string instead + return "" + } + // NSID must have at least two parts, verified by ParseNSID + parts = parts[:len(parts)-1] + // reverse + for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 { + parts[i], parts[j] = parts[j], parts[i] + } + return strings.ToLower(strings.Join(parts, ".")) +} + +func (n NSID) Name() string { + parts := strings.Split(string(n), ".") + return parts[len(parts)-1] +} + +func (n NSID) String() string { + return string(n) +} + +func (n NSID) Normalize() NSID { + parts := strings.Split(string(n), ".") + if len(parts) < 2 { + // something has gone wrong (would not validate); just return the whole identifier + return n + } + name := parts[len(parts)-1] + prefix := strings.ToLower(strings.Join(parts[:len(parts)-1], ".")) + return NSID(prefix + "." + name) +} + +func (n NSID) MarshalText() ([]byte, error) { + return []byte(n.String()), nil +} + +func (n *NSID) UnmarshalText(text []byte) error { + nsid, err := ParseNSID(string(text)) + if err != nil { + return err + } + *n = nsid + return nil +} diff --git a/atproto/syntax/nsid_test.go b/atproto/syntax/nsid_test.go new file mode 100644 index 000000000..224bc38ca --- /dev/null +++ b/atproto/syntax/nsid_test.go @@ -0,0 +1,76 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropNSIDsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/nsid_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseNSID(line) + if err != nil { + fmt.Println("FAILED, GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropNSIDsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/nsid_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseNSID(line) + if err == nil { + fmt.Println("FAILED, BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestNSIDParts(t *testing.T) { + assert := assert.New(t) + d, err := ParseNSID("cOm.ExAmple.blahFunc") + assert.NoError(err) + assert.Equal("example.com", d.Authority()) + assert.Equal("blahFunc", d.Name()) +} + +func TestNSIDNormalize(t *testing.T) { + assert := assert.New(t) + + nsid, err := ParseNSID("cOm.ExAmple.blahFunc") + assert.NoError(err) + assert.Equal("com.example.blahFunc", string(nsid.Normalize())) + assert.NoError(err) +} + +func TestNSIDNoPanic(t *testing.T) { + for _, s := range []string{"", ".", ".."} { + bad := NSID(s) + _ = bad.Authority() + _ = bad.Name() + _ = bad.Normalize() + } +} diff --git a/atproto/syntax/path.go b/atproto/syntax/path.go new file mode 100644 index 000000000..310bee17b --- /dev/null +++ b/atproto/syntax/path.go @@ -0,0 +1,26 @@ +package syntax + +import ( + "errors" + "fmt" + "strings" +) + +// Parses an atproto repo path string in to "collection" (NSID) and record key parts. +// +// Does not return partial success: either both collection and record key are complete (and error is nil), or both are empty string (and error is not nil) +func ParseRepoPath(raw string) (NSID, RecordKey, error) { + parts := strings.SplitN(raw, "/", 3) + if len(parts) != 2 { + return "", "", errors.New("expected path to have two parts, separated by single slash") + } + nsid, err := ParseNSID(parts[0]) + if err != nil { + return "", "", fmt.Errorf("collection part of path not a valid NSID: %w", err) + } + rkey, err := ParseRecordKey(parts[1]) + if err != nil { + return "", "", fmt.Errorf("record key part of path not valid: %w", err) + } + return nsid, rkey, nil +} diff --git a/atproto/syntax/path_test.go b/atproto/syntax/path_test.go new file mode 100644 index 000000000..e55f94fa7 --- /dev/null +++ b/atproto/syntax/path_test.go @@ -0,0 +1,41 @@ +package syntax + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRepoPath(t *testing.T) { + assert := assert.New(t) + + testValid := [][]string{ + {"app.bsky.feed.post/asdf", "app.bsky.feed.post", "asdf"}, + } + + testErr := []string{ + "", + "/", + "/app.bsky.feed.post/asdf", + "/asdf", + "./app.bsky.feed.post", + "blob/asdf", + "app.bsky.feed.post/", + "app.bsky.feed.post/.", + "app.bsky.feed.post/!", + } + + for _, parts := range testValid { + nsid, rkey, err := ParseRepoPath(parts[0]) + assert.NoError(err) + assert.Equal(parts[1], nsid.String()) + assert.Equal(parts[2], rkey.String()) + } + + for _, raw := range testErr { + nsid, rkey, err := ParseRepoPath(raw) + assert.Error(err) + assert.Equal("", nsid.String()) + assert.Equal("", rkey.String()) + } +} diff --git a/atproto/syntax/recordkey.go b/atproto/syntax/recordkey.go new file mode 100644 index 000000000..c8af66893 --- /dev/null +++ b/atproto/syntax/recordkey.go @@ -0,0 +1,48 @@ +package syntax + +import ( + "errors" + "regexp" +) + +var recordKeyRegex = regexp.MustCompile(`^[a-zA-Z0-9_~.:-]{1,512}$`) + +// String type which represents a syntaxtually valid RecordKey identifier, as could be included in an AT URI +// +// Always use [ParseRecordKey] instead of wrapping strings directly, especially when working with input. +// +// Syntax specification: https://atproto.com/specs/record-key +type RecordKey string + +func ParseRecordKey(raw string) (RecordKey, error) { + if raw == "" { + return "", errors.New("expected record key, got empty string") + } + if len(raw) > 512 { + return "", errors.New("recordkey is too long (512 chars max)") + } + if raw == "" || raw == "." || raw == ".." { + return "", errors.New("recordkey can not be empty, '.', or '..'") + } + if !recordKeyRegex.MatchString(raw) { + return "", errors.New("recordkey syntax didn't validate via regex") + } + return RecordKey(raw), nil +} + +func (r RecordKey) String() string { + return string(r) +} + +func (r RecordKey) MarshalText() ([]byte, error) { + return []byte(r.String()), nil +} + +func (r *RecordKey) UnmarshalText(text []byte) error { + rkey, err := ParseRecordKey(string(text)) + if err != nil { + return err + } + *r = rkey + return nil +} diff --git a/atproto/syntax/recordkey_test.go b/atproto/syntax/recordkey_test.go new file mode 100644 index 000000000..797ce3cfd --- /dev/null +++ b/atproto/syntax/recordkey_test.go @@ -0,0 +1,57 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropRecordKeysValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/recordkey_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseRecordKey(line) + if err != nil { + fmt.Println("FAILED, GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropRecordKeysInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/recordkey_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseRecordKey(line) + if err == nil { + fmt.Println("FAILED, BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestRecordKeyNoPanic(t *testing.T) { + for _, s := range []string{"", "a", ".", ".."} { + bad := RecordKey(s) + _ = bad.String() + } +} diff --git a/atproto/syntax/testdata/atidentifier_syntax_invalid.txt b/atproto/syntax/testdata/atidentifier_syntax_invalid.txt new file mode 100644 index 000000000..f0f84309b --- /dev/null +++ b/atproto/syntax/testdata/atidentifier_syntax_invalid.txt @@ -0,0 +1,28 @@ + +# invalid handles +did:thing.test +did:thing +john-.test +john.0 +john.- +xn--bcher-.tld +john..test +jo_hn.test + +# invalid DIDs +did +didmethodval +method:did:val +did:method: +didmethod:val +did:methodval) +:did:method:val +did:method:val: +did:method:val% +DID:method:val + +# other invalid stuff +email@example.com +@handle@example.com +@handle +blah diff --git a/atproto/syntax/testdata/atidentifier_syntax_valid.txt b/atproto/syntax/testdata/atidentifier_syntax_valid.txt new file mode 100644 index 000000000..cc4a42b0f --- /dev/null +++ b/atproto/syntax/testdata/atidentifier_syntax_valid.txt @@ -0,0 +1,15 @@ + +# allows valid handles +XX.LCS.MIT.EDU +john.test +jan.test +a234567890123456789.test +john2.test +john-john.test + +# allows valid DIDs +did:method:val +did:method:VAL +did:method:val123 +did:method:123 +did:method:val-two diff --git a/atproto/syntax/testdata/aturi_syntax_invalid.txt b/atproto/syntax/testdata/aturi_syntax_invalid.txt new file mode 100644 index 000000000..2ac2eadad --- /dev/null +++ b/atproto/syntax/testdata/aturi_syntax_invalid.txt @@ -0,0 +1,89 @@ + +# enforces spec basics +a://did:plc:asdf123 +at//did:plc:asdf123 +at:/a/did:plc:asdf123 +at:/did:plc:asdf123 +AT://did:plc:asdf123 +http://did:plc:asdf123 +://did:plc:asdf123 +at:did:plc:asdf123 +at:/did:plc:asdf123 +at:///did:plc:asdf123 +at://:/did:plc:asdf123 +at:/ /did:plc:asdf123 +at://did:plc:asdf123 +at://did:plc:asdf123/ + at://did:plc:asdf123 +at://did:plc:asdf123/com.atproto.feed.post +at://did:plc:asdf123/com.atproto.feed.post# +at://did:plc:asdf123/com.atproto.feed.post#/ +at://did:plc:asdf123/com.atproto.feed.post#/frag +at://did:plc:asdf123/com.atproto.feed.post#fr ag +//did:plc:asdf123 +at://name +at://name.0 +at://diD:plc:asdf123 +at://did:plc:asdf123/com.atproto.feed.p@st +at://did:plc:asdf123/com.atproto.feed.p$st +at://did:plc:asdf123/com.atproto.feed.p%st +at://did:plc:asdf123/com.atproto.feed.p&st +at://did:plc:asdf123/com.atproto.feed.p()t +at://did:plc:asdf123/com.atproto.feed_post +at://did:plc:asdf123/-com.atproto.feed.post +at://did:plc:asdf@123/com.atproto.feed.post +at://DID:plc:asdf123 +at://user.bsky.123 +at://bsky +at://did:plc: +at://did:plc: +at://frag + +# too long: 'at://did:plc:asdf123/com.atproto.feed.post/' + 'o'.repeat(8200) +at://did:plc:asdf123/com.atproto.feed.post/oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo + +# has specified behavior on edge cases +at://user.bsky.social// +at://user.bsky.social//com.atproto.feed.post +at://user.bsky.social/com.atproto.feed.post// +at://did:plc:asdf123/com.atproto.feed.post/asdf123/more/more', +at://did:plc:asdf123/short/stuff +at://did:plc:asdf123/12345 + +# enforces no trailing slashes +at://did:plc:asdf123/ +at://user.bsky.social/ +at://did:plc:asdf123/com.atproto.feed.post/ +at://did:plc:asdf123/com.atproto.feed.post/record/ +at://did:plc:asdf123/com.atproto.feed.post/record/#/frag + +# enforces strict paths +at://did:plc:asdf123/com.atproto.feed.post/asdf123/asdf + +# is very permissive about fragments +at://did:plc:asdf123# +at://did:plc:asdf123## +#at://did:plc:asdf123 +at://did:plc:asdf123#/asdf#/asdf + +# new less permissive about record keys for Lexicon use (with recordkey more specified) +at://did:plc:asdf123/com.atproto.feed.post/%23 +at://did:plc:asdf123/com.atproto.feed.post/$@!*)(:,;~.sdf123 +at://did:plc:asdf123/com.atproto.feed.post/~'sdf123") +at://did:plc:asdf123/com.atproto.feed.post/$ +at://did:plc:asdf123/com.atproto.feed.post/@ +at://did:plc:asdf123/com.atproto.feed.post/! +at://did:plc:asdf123/com.atproto.feed.post/* +at://did:plc:asdf123/com.atproto.feed.post/( +at://did:plc:asdf123/com.atproto.feed.post/, +at://did:plc:asdf123/com.atproto.feed.post/; +at://did:plc:asdf123/com.atproto.feed.post/abc%30123 +at://did:plc:asdf123/com.atproto.feed.post/%30 +at://did:plc:asdf123/com.atproto.feed.post/%3 +at://did:plc:asdf123/com.atproto.feed.post/% +at://did:plc:asdf123/com.atproto.feed.post/%zz +at://did:plc:asdf123/com.atproto.feed.post/%%% + +# disallow dot / double-dot +at://did:plc:asdf123/com.atproto.feed.post/. +at://did:plc:asdf123/com.atproto.feed.post/.. diff --git a/atproto/syntax/testdata/aturi_syntax_valid.txt b/atproto/syntax/testdata/aturi_syntax_valid.txt new file mode 100644 index 000000000..be30ff4bd --- /dev/null +++ b/atproto/syntax/testdata/aturi_syntax_valid.txt @@ -0,0 +1,35 @@ + +# enforces spec basics +at://did:plc:asdf123 +at://user.bsky.social +at://did:plc:asdf123/com.atproto.feed.post +at://did:plc:asdf123/com.atproto.feed.post/record + +# very long: 'at://did:plc:asdf123/com.atproto.feed.post/' + 'o'.repeat(512) +at://did:plc:asdf123/com.atproto.feed.post/oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo + +# enforces no trailing slashes +at://did:plc:asdf123 +at://user.bsky.social +at://did:plc:asdf123/com.atproto.feed.post +at://did:plc:asdf123/com.atproto.feed.post/record + +# enforces strict paths +at://did:plc:asdf123/com.atproto.feed.post/asdf123 + +# is very permissive about record keys +at://did:plc:asdf123/com.atproto.feed.post/asdf123 +at://did:plc:asdf123/com.atproto.feed.post/a + +at://did:plc:asdf123/com.atproto.feed.post/asdf-123 +at://did:abc:123 +at://did:abc:123/io.nsid.someFunc/record-key + +at://did:abc:123/io.nsid.someFunc/self. +at://did:abc:123/io.nsid.someFunc/lang: +at://did:abc:123/io.nsid.someFunc/: +at://did:abc:123/io.nsid.someFunc/- +at://did:abc:123/io.nsid.someFunc/_ +at://did:abc:123/io.nsid.someFunc/~ +at://did:abc:123/io.nsid.someFunc/... +at://did:plc:asdf123/com.atproto.feed.postV2 diff --git a/atproto/syntax/testdata/cid_syntax_invalid.txt b/atproto/syntax/testdata/cid_syntax_invalid.txt new file mode 100644 index 000000000..5bd1d007c --- /dev/null +++ b/atproto/syntax/testdata/cid_syntax_invalid.txt @@ -0,0 +1,16 @@ +example.com +https://example.com +cid:bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi +. +12345 + +# whitespace + bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi +bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi +bafybe igdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi + +# old CIDv0 not supported +QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR + +# https://github.com/ipfs-shipyard/is-ipfs/blob/master/test/test-cid.spec.ts +noop diff --git a/atproto/syntax/testdata/cid_syntax_valid.txt b/atproto/syntax/testdata/cid_syntax_valid.txt new file mode 100644 index 000000000..9dd3dfed8 --- /dev/null +++ b/atproto/syntax/testdata/cid_syntax_valid.txt @@ -0,0 +1,14 @@ + +# examples from https://docs.ipfs.tech/concepts/content-addressing +bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi + +# https://github.com/ipfs-shipyard/is-ipfs/blob/master/test/test-cid.spec.ts +zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 +bafybeie5gq4jxvzmsym6hjlwxej4rwdoxt7wadqvmmwbqi7r27fclha2va + +# more contrived examples +mBcDxtdWx0aWhhc2g+ +z7x3CtScH765HvShXT +zdj7WhuEjrB52m1BisYCtmjH1hSKa7yZ3jEZ9JcXaFRD51wVz +7134036155352661643226414134664076 +f017012202c5f688262e0ece8569aa6f94d60aad55ca8d9d83734e4a7430d0cff6588ec2b diff --git a/atproto/syntax/testdata/datetime_parse_invalid.txt b/atproto/syntax/testdata/datetime_parse_invalid.txt new file mode 100644 index 000000000..3b3031c6a --- /dev/null +++ b/atproto/syntax/testdata/datetime_parse_invalid.txt @@ -0,0 +1,10 @@ +# superficial syntax parses ok, but are not valid datetimes for semantic reasons (eg, "month zero") +1985-00-12T23:20:50.123Z +1985-04-00T23:20:50.123Z +1985-13-12T23:20:50.123Z +1985-04-12T25:20:50.123Z +1985-04-12T23:99:50.123Z +1985-04-12T23:20:61.123Z + +# ISO-8601, but normalizes to a negative time +0000-01-01T00:00:00+01:00 diff --git a/atproto/syntax/testdata/datetime_syntax_invalid.txt b/atproto/syntax/testdata/datetime_syntax_invalid.txt new file mode 100644 index 000000000..9db91eb6d --- /dev/null +++ b/atproto/syntax/testdata/datetime_syntax_invalid.txt @@ -0,0 +1,66 @@ + +# subtle changes to: 1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.123z +01985-04-12T23:20:50.123Z +985-04-12T23:20:50.123Z +1985-04-12T23:20:50.Z +1985-04-32T23;20:50.123Z +1985-04-32T23;20:50.123Z + +# en-dash and em-dash +1985—04-32T23;20:50.123Z +1985–04-32T23;20:50.123Z + +# whitespace + 1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.123Z +1985-04-12T 23:20:50.123Z + +# not enough zero padding +1985-4-12T23:20:50.123Z +1985-04-2T23:20:50.123Z +1985-04-12T3:20:50.123Z +1985-04-12T23:0:50.123Z +1985-04-12T23:20:5.123Z + +# too much zero padding +01985-04-12T23:20:50.123Z +1985-004-12T23:20:50.123Z +1985-04-012T23:20:50.123Z +1985-04-12T023:20:50.123Z +1985-04-12T23:020:50.123Z +1985-04-12T23:20:050.123Z + +# strict capitalization (ISO-8601) +1985-04-12t23:20:50.123Z +1985-04-12T23:20:50.123z + +# RFC-3339, but not ISO-8601 +1985-04-12T23:20:50.123-00:00 +1985-04-12_23:20:50.123Z +1985-04-12 23:20:50.123Z + +# ISO-8601, but weird +1985-04-274T23:20:50.123Z + +# timezone is required +1985-04-12T23:20:50.123 +1985-04-12T23:20:50 + +1985-04-12 +1985-04-12T23:20Z +1985-04-12T23:20:5Z +1985-04-12T23:20:50.123 ++001985-04-12T23:20:50.123Z +23:20:50.123Z + +1985-04-12T23:20:50.123+00 +1985-04-12T23:20:50.123+00:0 +1985-04-12T23:20:50.123+0:00 +1985-04-12T23:20:50.123 +1985-04-12T23:20:50.123+0000 +1985-04-12T23:20:50.123+00 +1985-04-12T23:20:50.123+ +1985-04-12T23:20:50.123- + +-000001-12-31T23:00:00.000Z diff --git a/atproto/syntax/testdata/datetime_syntax_valid.txt b/atproto/syntax/testdata/datetime_syntax_valid.txt new file mode 100644 index 000000000..18c3a4253 --- /dev/null +++ b/atproto/syntax/testdata/datetime_syntax_valid.txt @@ -0,0 +1,42 @@ +# "preferred" +1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.000Z +2000-01-01T00:00:00.000Z +1985-04-12T23:20:50.123456Z +1985-04-12T23:20:50.120Z +1985-04-12T23:20:50.120000Z + +# "supported" +1985-04-12T23:20:50.1235678912345Z +1985-04-12T23:20:50.100Z +1985-04-12T23:20:50Z +1985-04-12T23:20:50.0Z +1985-04-12T23:20:50.123+00:00 +1985-04-12T23:20:50.123-07:00 +1985-04-12T23:20:50.123+07:00 +1985-04-12T23:20:50.123+01:45 +0985-04-12T23:20:50.123-07:00 +1985-04-12T23:20:50.123-07:00 +0123-01-01T00:00:00.000Z + +# various precisions, up through at least 12 digits +1985-04-12T23:20:50.1Z +1985-04-12T23:20:50.12Z +1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.1234Z +1985-04-12T23:20:50.12345Z +1985-04-12T23:20:50.123456Z +1985-04-12T23:20:50.1234567Z +1985-04-12T23:20:50.12345678Z +1985-04-12T23:20:50.123456789Z +1985-04-12T23:20:50.1234567890Z +1985-04-12T23:20:50.12345678901Z +1985-04-12T23:20:50.123456789012Z + +# extreme but currently allowed +0000-01-01T00:00:00.000Z +0001-01-01T00:00:00.000Z +0010-12-31T23:00:00.000Z +1000-12-31T23:00:00.000Z +1900-12-31T23:00:00.000Z +3001-12-31T23:00:00.000Z diff --git a/atproto/syntax/testdata/did_syntax_invalid.txt b/atproto/syntax/testdata/did_syntax_invalid.txt new file mode 100644 index 000000000..9e724b3d7 --- /dev/null +++ b/atproto/syntax/testdata/did_syntax_invalid.txt @@ -0,0 +1,19 @@ +did +didmethodval +method:did:val +did:method: +didmethod:val +did:methodval) +:did:method:val +did.method.val +did:method:val: +did:method:val% +DID:method:val +did:METHOD:val +did:m123:val +did:method:val/two +did:method:val?two +did:method:val#two +did:method:val% +did:method:vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + diff --git a/atproto/syntax/testdata/did_syntax_valid.txt b/atproto/syntax/testdata/did_syntax_valid.txt new file mode 100644 index 000000000..5aa6c9e7e --- /dev/null +++ b/atproto/syntax/testdata/did_syntax_valid.txt @@ -0,0 +1,26 @@ +did:method:val +did:method:VAL +did:method:val123 +did:method:123 +did:method:val-two +did:method:val_two +did:method:val.two +did:method:val:two +did:method:val%BB +did:method:vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +did:m:v +did:method::::val +did:method:- +did:method:-:_:.:%ab +did:method:. +did:method:_ +did:method::. + +# allows some real DID values +did:onion:2gzyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid +did:example:123456789abcdefghi +did:plc:7iza6de2dwap2sbkpav7c6c6 +did:web:example.com +did:web:localhost%3A1234 +did:key:zQ3shZc2QzApp2oymGvQbzP8eKheVshBHbU4ZYjeXqwSKEn6N +did:ethr:0xb9c5714089478a327f09197987f16f9e5d936e8a diff --git a/atproto/syntax/testdata/handle_syntax_invalid.txt b/atproto/syntax/testdata/handle_syntax_invalid.txt new file mode 100644 index 000000000..49275a390 --- /dev/null +++ b/atproto/syntax/testdata/handle_syntax_invalid.txt @@ -0,0 +1,61 @@ +# throws on invalid handles +did:thing.test +did:thing +john-.test +john.0 +john.- +xn--bcher-.tld +john..test +jo_hn.test +-john.test +.john.test +jo!hn.test +jo%hn.test +jo&hn.test +jo@hn.test +jo*hn.test +jo|hn.test +jo:hn.test +jo/hn.test +john💩.test +bücher.test +john .test +john.test. +john +john. +.john +john.test. +.john.test + john.test +john.test +joh-.test +john.-est +john.tes- + +# max over all handle: 'shoooort' + '.loooooooooooooooooooooooooong'.repeat(9) + '.test' +shoooort.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.test + +# max segment: 'short.' + 'o'.repeat(64) + '.test' +short.oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.test + +# throws on "dotless" TLD handles +org +ai +gg +io + +# correctly validates corner cases (modern vs. old RFCs) +cn.8 +thing.0aa +thing.0aa + +# does not allow IP addresses as handles +127.0.0.1 +192.168.0.142 +fe80::7325:8a97:c100:94b +2600:3c03::f03c:9100:feb0:af1f + +# examples from stackoverflow +-notvalid.at-all +-thing.com +www.masełkowski.pl.com diff --git a/atproto/syntax/testdata/handle_syntax_valid.txt b/atproto/syntax/testdata/handle_syntax_valid.txt new file mode 100644 index 000000000..a23a92138 --- /dev/null +++ b/atproto/syntax/testdata/handle_syntax_valid.txt @@ -0,0 +1,90 @@ +# allows valid handles +A.ISI.EDU +XX.LCS.MIT.EDU +SRI-NIC.ARPA +john.test +jan.test +a234567890123456789.test +john2.test +john-john.test +john.bsky.app +jo.hn +a.co +a.org +joh.n +j0.h0 +jaymome-johnber123456.test +jay.mome-johnber123456.test +john.test.bsky.app + +# max over all handle: 'shoooort' + '.loooooooooooooooooooooooooong'.repeat(8) + '.test' +shoooort.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.loooooooooooooooooooooooooong.test + +# max segment: 'short.' + 'o'.repeat(63) + '.test' +short.ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.test + +# NOTE: this probably isn't ever going to be a real domain, but my read of the RFC is that it would be possible +john.t + +# allows .local and .arpa handles (proto-level) +laptop.local +laptop.arpa + +# allows punycode handles +# 💩.test +xn--ls8h.test +# bücher.tld +xn--bcher-kva.tld +xn--3jk.com +xn--w3d.com +xn--vqb.com +xn--ppd.com +xn--cs9a.com +xn--8r9a.com +xn--cfd.com +xn--5jk.com +xn--2lb.com + +# allows onion (Tor) handles +expyuzz4wqqyqhjn.onion +friend.expyuzz4wqqyqhjn.onion +g2zyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.onion +friend.g2zyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.onion +friend.g2zyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.onion +2gzyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.onion +friend.2gzyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.onion + +# correctly validates corner cases (modern vs. old RFCs) +12345.test +8.cn +4chan.org +4chan.o-g +blah.4chan.org +thing.a01 +120.0.0.1.com +0john.test +9sta--ck.com +99stack.com +0ohn.test +john.t--t +thing.0aa.thing + +# examples from stackoverflow +stack.com +sta-ck.com +sta---ck.com +sta--ck9.com +stack99.com +sta99ck.com +google.com.uk +google.co.in +google.com +maselkowski.pl +m.maselkowski.pl +xn--masekowski-d0b.pl +xn--fiqa61au8b7zsevnm8ak20mc4a87e.xn--fiqs8s +xn--stackoverflow.com +stackoverflow.xn--com +stackoverflow.co.uk +xn--masekowski-d0b.pl +xn--fiqa61au8b7zsevnm8ak20mc4a87e.xn--fiqs8s diff --git a/atproto/syntax/testdata/language_syntax_invalid.txt b/atproto/syntax/testdata/language_syntax_invalid.txt new file mode 100644 index 000000000..9bba437a2 --- /dev/null +++ b/atproto/syntax/testdata/language_syntax_invalid.txt @@ -0,0 +1,10 @@ +jaja +. +123 +JA +j +ja- +a-DE + +# technically not valid, but allowing in naive parser +#de-419-DE diff --git a/atproto/syntax/testdata/language_syntax_valid.txt b/atproto/syntax/testdata/language_syntax_valid.txt new file mode 100644 index 000000000..8a4a5fee8 --- /dev/null +++ b/atproto/syntax/testdata/language_syntax_valid.txt @@ -0,0 +1,18 @@ +ja +ban +pt-BR +hy-Latn-IT-arevela +en-GB +zh-Hant +sgn-BE-NL +es-419 +en-GB-boont-r-extended-sequence-x-private + +# grandfathered +zh-hakka +i-default +i-navajo + +# https://github.com/sebinsua/ietf-language-tag-regex/blob/master/test.js +de-CH-1901 +qaa-Qaaa-QM-x-southern diff --git a/atproto/syntax/testdata/nsid_syntax_invalid.txt b/atproto/syntax/testdata/nsid_syntax_invalid.txt new file mode 100644 index 000000000..557c75e06 --- /dev/null +++ b/atproto/syntax/testdata/nsid_syntax_invalid.txt @@ -0,0 +1,30 @@ +# length checks +com.oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.foo +com.example.oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo +com.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.foo + +# invalid examples +com.example.foo.* +com.example.foo.blah* +com.example.foo.*blah +com.exa💩ple.thing +a-0.b-1.c-3 +a-0.b-1.c-o +1.0.0.127.record +0two.example.foo +example.com +com.example +a. +.one.two.three +one.two.three +one.two..three +one .two.three + one.two.three +com.exa💩ple.thing +com.atproto.feed.p@st +com.atproto.feed.p_st +com.atproto.feed.p*st +com.atproto.feed.po#t +com.atproto.feed.p!ot +com.example-.foo +com.example.fooBar.2 diff --git a/atproto/syntax/testdata/nsid_syntax_valid.txt b/atproto/syntax/testdata/nsid_syntax_valid.txt new file mode 100644 index 000000000..7ee4cec6d --- /dev/null +++ b/atproto/syntax/testdata/nsid_syntax_valid.txt @@ -0,0 +1,32 @@ +# length checks +com.ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo.foo +com.example.ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo +com.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.middle.foo + +# valid examples +com.example.fooBar +com.example.fooBarV2 +net.users.bob.ping +a.b.c +m.xn--masekowski-d0b.pl +one.two.three +one.two.three.four-and.FiVe +one.2.three +a-0.b-1.c +a0.b1.cc +cn.8.lex.stuff +test.12345.record +a01.thing.record +a.0.c +xn--fiqs8s.xn--fiqa61au8b7zsevnm8ak20mc4a87e.record.two +a0.b1.c3 +com.example.f00 + +# allows onion (Tor) NSIDs +onion.expyuzz4wqqyqhjn.spec.getThing +onion.g2zyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.lex.deleteThing + +# allows starting-with-numeric segments (same as domains) +org.4chan.lex.getThing +cn.8.lex.stuff +onion.2gzyxa5ihm7nsggfxnu52rck2vv4rvmdlkiu3zzui5du4xyclen53wid.lex.deleteThing diff --git a/atproto/syntax/testdata/recordkey_syntax_invalid.txt b/atproto/syntax/testdata/recordkey_syntax_invalid.txt new file mode 100644 index 000000000..52106d873 --- /dev/null +++ b/atproto/syntax/testdata/recordkey_syntax_invalid.txt @@ -0,0 +1,15 @@ +# specs +alpha/beta +. +.. +#extra +@handle +any space +any+space +number[3] +number(3) +"quote" +dHJ1ZQ== + +# too long: 'o'.repeat(513) +ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo diff --git a/atproto/syntax/testdata/recordkey_syntax_valid.txt b/atproto/syntax/testdata/recordkey_syntax_valid.txt new file mode 100644 index 000000000..92e8b7e31 --- /dev/null +++ b/atproto/syntax/testdata/recordkey_syntax_valid.txt @@ -0,0 +1,21 @@ +# specs +self +example.com +~1.2-3_ +dHJ1ZQ +_ +literal:self +pre:fix + +# more corner-cases +: +- +_ +~ +... +self. +lang: +:lang + +# very long: 'o'.repeat(512) +oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo diff --git a/atproto/syntax/testdata/tid_syntax_invalid.txt b/atproto/syntax/testdata/tid_syntax_invalid.txt new file mode 100644 index 000000000..30bee75be --- /dev/null +++ b/atproto/syntax/testdata/tid_syntax_invalid.txt @@ -0,0 +1,19 @@ + +# not base32 +3jzfcijpj2z21 +0000000000000 + +# case-sensitive +3JZFCIJPJ2Z2A + +# too long/short +3jzfcijpj2z2aa +3jzfcijpj2z2 +222 + +# old dashes syntax not actually supported (TTTT-TTT-TTTT-CC) +3jzf-cij-pj2z-2a + +# high bit can't be high +zzzzzzzzzzzzz +kjzfcijpj2z2a diff --git a/atproto/syntax/testdata/tid_syntax_valid.txt b/atproto/syntax/testdata/tid_syntax_valid.txt new file mode 100644 index 000000000..93be7ee82 --- /dev/null +++ b/atproto/syntax/testdata/tid_syntax_valid.txt @@ -0,0 +1,7 @@ +# 13 digits +# 234567abcdefghijklmnopqrstuvwxyz + +3jzfcijpj2z2a +7777777777777 +3zzzzzzzzzzzz +2222222222222 diff --git a/atproto/syntax/testdata/uri_syntax_invalid.txt b/atproto/syntax/testdata/uri_syntax_invalid.txt new file mode 100644 index 000000000..f79a4ba12 --- /dev/null +++ b/atproto/syntax/testdata/uri_syntax_invalid.txt @@ -0,0 +1,17 @@ + +example.com +://example.com +//example.com +http: +.http://example.com +-http://example.com +12345 +127.0.0.1 + +https://example.com/path gap + https://example.com/path +https://example.com/trailing-whitespace + +# too long (max 8 kbytes) +# python: "https://example.com/" + 8200 *"x" +https://example.com/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx diff --git a/atproto/syntax/testdata/uri_syntax_valid.txt b/atproto/syntax/testdata/uri_syntax_valid.txt new file mode 100644 index 000000000..709a6136a --- /dev/null +++ b/atproto/syntax/testdata/uri_syntax_valid.txt @@ -0,0 +1,14 @@ + +https://example.com +https://example.com/path?q=blah&yes=true#frag.123 +dns:example.com +at://handle.example.com/nsid/rkey +did:key:zQ3shZc2QzApp2oymGvQbzP8eKheVshBHbU4ZYjeXqwSKEn6N +content-type:text/plan +microsoft.windows.camera:thing +go://?Mercedes%20Benz + +# long (but not too long) +# python: "https://example.com/" + 5000*"x" +https://example.com/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + diff --git a/atproto/syntax/tid.go b/atproto/syntax/tid.go new file mode 100644 index 000000000..17cfdf4ae --- /dev/null +++ b/atproto/syntax/tid.go @@ -0,0 +1,150 @@ +package syntax + +import ( + "encoding/base32" + "errors" + "regexp" + "strings" + "sync" + "time" +) + +const ( + Base32SortAlphabet = "234567abcdefghijklmnopqrstuvwxyz" +) + +func Base32Sort() *base32.Encoding { + return base32.NewEncoding(Base32SortAlphabet).WithPadding(base32.NoPadding) +} + +// Represents a TID in string format, as would pass Lexicon syntax validation. +// +// Always use [ParseTID] instead of wrapping strings directly, especially when working with network input. +// +// Syntax specification: https://atproto.com/specs/record-key +type TID string + +var tidRegex = regexp.MustCompile(`^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$`) + +func ParseTID(raw string) (TID, error) { + if raw == "" { + return "", errors.New("expected TID, got empty string") + } + if len(raw) != 13 { + return "", errors.New("TID is wrong length (expected 13 chars)") + } + if !tidRegex.MatchString(raw) { + return "", errors.New("TID syntax didn't validate via regex") + } + return TID(raw), nil +} + +// Naive (unsafe) one-off TID generation with the current time. +// +// You should usually use a [TIDClock] to ensure monotonic output. +func NewTIDNow(clockId uint) TID { + return NewTID(time.Now().UTC().UnixMicro(), clockId) +} + +func NewTIDFromInteger(v uint64) TID { + v = (0x7FFF_FFFF_FFFF_FFFF & v) + s := "" + for i := 0; i < 13; i++ { + s = string(Base32SortAlphabet[v&0x1F]) + s + v = v >> 5 + } + return TID(s) +} + +// Constructs a new TID from a UNIX timestamp (in milliseconds) and clock ID value. +func NewTID(unixMicros int64, clockId uint) TID { + v := (uint64(unixMicros&0x1F_FFFF_FFFF_FFFF) << 10) | uint64(clockId&0x3FF) + return NewTIDFromInteger(v) +} + +// Constructs a new TID from a [time.Time] and clock ID value +func NewTIDFromTime(ts time.Time, clockId uint) TID { + return NewTID(ts.UTC().UnixMicro(), clockId) +} + +// Returns full integer representation of this TID (not used often) +func (t TID) Integer() uint64 { + s := t.String() + if len(s) != 13 { + return 0 + } + var v uint64 + for i := 0; i < 13; i++ { + c := strings.IndexByte(Base32SortAlphabet, s[i]) + if c < 0 { + return 0 + } + v = (v << 5) | uint64(c&0x1F) + } + return v +} + +// Returns the golang [time.Time] corresponding to this TID's timestamp. +func (t TID) Time() time.Time { + i := t.Integer() + i = (i >> 10) & 0x1FFF_FFFF_FFFF_FFFF + return time.UnixMicro(int64(i)).UTC() +} + +// Returns the clock ID part of this TID, as an unsigned integer +func (t TID) ClockID() uint { + i := t.Integer() + return uint(i & 0x3FF) +} + +func (t TID) String() string { + return string(t) +} + +func (t TID) MarshalText() ([]byte, error) { + return []byte(t.String()), nil +} + +func (t *TID) UnmarshalText(text []byte) error { + tid, err := ParseTID(string(text)) + if err != nil { + return err + } + *t = tid + return nil +} + +// TID generator, which keeps state to ensure TID values always monotonically increase. +// +// Uses [sync.Mutex], so may block briefly but safe for concurrent use. +type TIDClock struct { + ClockID uint + mtx sync.Mutex + lastUnixMicro int64 +} + +func NewTIDClock(clockId uint) TIDClock { + return TIDClock{ + ClockID: clockId, + } +} + +func ClockFromTID(t TID) TIDClock { + um := t.Integer() + um = (um >> 10) & 0x1FFF_FFFF_FFFF_FFFF + return TIDClock{ + ClockID: t.ClockID(), + lastUnixMicro: int64(um), + } +} + +func (c *TIDClock) Next() TID { + now := time.Now().UTC().UnixMicro() + c.mtx.Lock() + if now <= c.lastUnixMicro { + now = c.lastUnixMicro + 1 + } + c.lastUnixMicro = now + c.mtx.Unlock() + return NewTID(now, c.ClockID) +} diff --git a/atproto/syntax/tid_test.go b/atproto/syntax/tid_test.go new file mode 100644 index 000000000..ed0d94bee --- /dev/null +++ b/atproto/syntax/tid_test.go @@ -0,0 +1,131 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestInteropTIDsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/tid_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseTID(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropTIDsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/tid_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseTID(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestTIDParts(t *testing.T) { + assert := assert.New(t) + + raw := "3kao2cl6lyj2p" + tid, err := ParseTID(raw) + assert.NoError(err) + // TODO: assert.Equal(uint64(0x181a8044491f3bec), tid.Integer()) + // TODO: assert.Equal(uint(1004), tid.ClockID()) + assert.Equal(2023, tid.Time().Year()) + + out := NewTID(tid.Time().UnixMicro(), tid.ClockID()) + assert.Equal(raw, out.String()) + assert.Equal(tid.ClockID(), out.ClockID()) + assert.Equal(tid.Time(), out.Time()) + assert.Equal(tid.Integer(), out.Integer()) + + out2 := NewTIDFromInteger(tid.Integer()) + assert.Equal(tid.Integer(), out2.Integer()) +} + +func TestTIDExamples(t *testing.T) { + assert := assert.New(t) + // TODO: seems like TS code might be wrong? "242k52k4kg3s2" + assert.Equal("242k52k4kg3sc", NewTIDFromInteger(0x0102030405060708).String()) + assert.Equal(uint64(0x0102030405060708), TID("242k52k4kg3sc").Integer()) + //assert.Equal("2222222222222", NewTIDFromInteger(0x0000000000000000).String()) + //assert.Equal(uint64(), TID("242k52k4kg3s2").Integer()) + assert.Equal("2222222222223", NewTIDFromInteger(0x0000000000000001).String()) + assert.Equal(uint64(0x0000000000000001), TID("2222222222223").Integer()) + + assert.Equal("6222222222222", NewTIDFromInteger(0x4000000000000000).String()) + assert.Equal(uint64(0x4000000000000000), TID("6222222222222").Integer()) + + // ignoring type byte + assert.Equal("2222222222222", NewTIDFromInteger(0x8000000000000000).String()) +} + +func TestTIDNoPanic(t *testing.T) { + for _, s := range []string{"", "3jzfcijpj2z2aa", "3jzfcijpj2z2", ".."} { + bad := TID(s) + _ = bad.ClockID() + _ = bad.Integer() + _ = bad.Time() + _ = bad.String() + } +} + +func TestTIDConstruction(t *testing.T) { + assert := assert.New(t) + + zero := NewTID(0, 0) + assert.Equal("2222222222222", zero.String()) + assert.Equal(uint64(0), zero.Integer()) + assert.Equal(uint(0), zero.ClockID()) + assert.Equal(time.UnixMicro(0).UTC(), zero.Time()) + + now := NewTIDNow(1011) + assert.Equal(uint(1011), now.ClockID()) + assert.True(time.Since(now.Time()) < time.Minute) + + over := NewTIDNow(4096) + assert.Equal(uint(0), over.ClockID()) + + next := NewTIDFromTime(time.Now(), 123) + assert.Equal(uint(123), next.ClockID()) + assert.True(time.Since(next.Time()) < time.Minute) +} + +func TestTIDClock(t *testing.T) { + assert := assert.New(t) + + clk := NewTIDClock(0) + last := NewTID(0, 0) + for i := 0; i < 100; i++ { + next := clk.Next() + assert.Greater(next, last) + last = next + } +} diff --git a/atproto/syntax/uri.go b/atproto/syntax/uri.go new file mode 100644 index 000000000..fbf8807c4 --- /dev/null +++ b/atproto/syntax/uri.go @@ -0,0 +1,44 @@ +package syntax + +import ( + "errors" + "regexp" +) + +// Represents an arbitrary URI in string format, as would pass Lexicon syntax validation. +// +// The syntax is minimal and permissive, designed for fast verification and exact-string passthrough, not schema-specific parsing or validation. For example, will not validate AT-URI or DID strings. +// +// Always use [ParseURI] instead of wrapping strings directly, especially when working with network input. +type URI string + +func ParseURI(raw string) (URI, error) { + if raw == "" { + return "", errors.New("expected URI, got empty string") + } + if len(raw) > 8192 { + return "", errors.New("URI is too long (8192 chars max)") + } + var uriRegex = regexp.MustCompile(`^[a-z][a-z.-]{0,80}:[[:graph:]]+$`) + if !uriRegex.MatchString(raw) { + return "", errors.New("URI syntax didn't validate via regex") + } + return URI(raw), nil +} + +func (u URI) String() string { + return string(u) +} + +func (u URI) MarshalText() ([]byte, error) { + return []byte(u.String()), nil +} + +func (u *URI) UnmarshalText(text []byte) error { + uri, err := ParseURI(string(text)) + if err != nil { + return err + } + *u = uri + return nil +} diff --git a/atproto/syntax/uri_test.go b/atproto/syntax/uri_test.go new file mode 100644 index 000000000..cca4a6d7f --- /dev/null +++ b/atproto/syntax/uri_test.go @@ -0,0 +1,50 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropURIsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/uri_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseURI(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropURIsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/uri_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseURI(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} diff --git a/automod/HOWTO_write_rules.md b/automod/HOWTO_write_rules.md new file mode 100644 index 000000000..12ae8c555 --- /dev/null +++ b/automod/HOWTO_write_rules.md @@ -0,0 +1,150 @@ + +HOWTO: Write automod Rules +========================== + +The short version is: + +- identity a behavior pattern or type of content in the network, and an action that should be taken in response +- write a "rule" function, in golang, which will detect this pattern (usually start by copying an existing rule) +- register the new rule with the rule engine +- test triggering the rule, either in a test network or using "captured" content from a real network +- deploy the rule, first with reduced "effects" (actions) to monitor impact + +The `automod/rules` package contains a set of example rules and some shared helper functions, and demonstrates some patterns for how to use counters, sets, filters, and account metadata to compose a rule pattern. + +## How Rules Work + +Automod rules are golang functions which get called every time a relevant event takes place in the network. Rule functions receive static metadata about the event; can fetch additional state or metadata as needed; and can optionally output "effects". These effects can include state mutations (such as incrementing counters), or taking moderation actions. + +There are multiple rule function types (eg, specifically for bsky "posts", or for atproto identity updates), but they all receive a `c` "Context" argument as the primary API for the rules system, including both accessing metadata and recording effects. + +Multiple rules for the same event may be run concurrently, or in arbitrary order. Effects *are not* visible between rule execution on the same event, and are only persisted after all rules have finished executing. This means that if one rule increments a counter or adds a label, other rules will not "see" that effect when processing the same event. + +Effects are automatically de-duplicated by the rules engine, both between concurrent rules and against the current state of an effect's subject. This means that rules can generally "trigger" continuously (eg, report an account on the basis of multiple posts), and the action will only take place once (not reported multiple times). + +It is expected that some rules will act together, for example paired rules on record creation and record deletion. + +The design philosophy of rules are that they mostly contain their own configuration, as code. Rules are not expected to be directly configurable, and changing the "effects" or action of a rule is a change to the rule code itself. + + +## Rule APIs + +There are two general categories of rules and effects: at the account-level, and at the record-level, with the later being a superset of the former. + +Note that none of the Context methods return errors. If errors are encountered (for example, network faults), error state is persisted internally to the Context object, a placeholder value is returned, and no effects will be persisted for the overall event execution. This is to keep rule code simple and readable. + + +### Rule Types + +The notable rule function types are: + +- `type IdentityRuleFunc = func(c *AccountContext) error`: triggers on events like handle updates or account migrations +- `type RecordRuleFunc = func(c *RecordContext) error`: triggers on every repo operation: create, update, or delete. Triggers for every record type, including posts and profiles +- `type PostRuleFunc = func(c *RecordContext, post *appbsky.FeedPost) error`: triggers on creation or update of any `app.bsky.feed.post` record. The post record is de-serialized for convenience, but otherwise this is basically just `RecordRuleFunc` +- `type ProfileRuleFunc = func(c *RecordContext, profile *appbsky.ActorProfile) error`: same as `PostRuleFunc`, but for profile + +The `PostRuleFunc` and `ProfileRuleFunc` are simply affordances so that rules for those common record types don't all need to filter and type-cast. Rules for other record types (such as `app.bsky.graph.follow`) do need to use `RecordRuleFunc` and implement that filtering and type-casting. + +### Pre-Hydrated Metadata + +The `c *automod.AccountContext` parameter provides the following pre-hydrated metadata: + +- `c.Account.Identity`: atproto identity for the account, including `DID` and `Handle` fields, and the PDS endpoint URL (if declared) +- `c.Account.Private` (optional): contains things like `.IndexedAt` (account first seen), `.Email` (the current registered account email), and `.EmailConfirmed` (boolean). Only hydrated when the rule engine is configured with admin privileges, and the account is on a PDS those privileges have access to +- `c.Account.Profile` (optional): a cached subset of the account's bsky profile record +- `c.Account.AccountLabels` (array of strings): cached view of any moderation labels applied to the account, by the relevant "local" moderation service +- `c.Account.AccountNegatedLabels` (array of strings) +- `c.Account.Takendown` (bool): if the account is currently taken down or not +- `c.Account.FollowersCount` (int64): cached +- `c.Account.PostsCount` (int64): cached + +The `c *automod.RecordContext` parameter is a superset of `AccountContext` and also includes: + +- `c.RecordOp.Action`: one of "create", "update", or "delete" +- `c.RecordOp.DID` +- `c.RecordOp.Collection` +- `c.RecordOp.RecordKey` +- `c.RecordOp.CID` (optional): not included for "delete" +- `c.RecordOp.Value` (optional): the record itself, usually as a pointer to an un-marshalled struct + +### Counters + +All `Context` objects provide access to counters. Rules don't need to pre-configure counter namespaces or values, they can just start using them. The default value for a counter which has never been incremented is `0`. + +The datastore providing counters is an internal implementation/configuration detail of the rule engine, but is usually Redis. Reads (`GetCount`) may hit the network but are pretty fast. + +Incrementing a counter is an "effect" and is not persisted until the end of all rule execution for an event. That is, if you read, increment, and read again, you will read the same count. + +The counter API has distinct "namespace" and "value" fields, which are combined to form a key. You generally chose a unique namespace specific to your rule and counter type, and then values are either a fixed string or a normalized field like a DID or hash. The keyspace is global, so rules can access and mutate each other's counters, and need to avoid namespace collisions. + +Time periods for counters: + +- `automod.PeriodHour`: time bucket of current hour +- `automod.PeriodDay`: time bucket of current day +- `automod.PeriodTotal`: all-time counts + +Basic counters: + +- `c.GetCount(, , )`: reads count for the specific time period +- `c.Increment(, )`: increments all time periods +- `c.IncrementPeriod(, , )`: increments only a single time period bucket, as a resource optimization. You should generally use the full `Increment` method. + +"Distinct value" counters use a statistical data structure (hyperloglog) to estimate the number of unique strings incremented for the given bucket. These counters consume more memory (up to a couple KBytes per counter), though they are generally smaller for small-N buckets. + +- `c.GetCountDistinct(, , )` +- `c.IncrementDistinct(, , )` + +### Sets + +Sets are a mechanism to separate configuration from rule implementation. They are simply named arrays of strings. Membership checks are very fast, and won't hit the network more than once per set per rule invocation. + +- `c.InSet(, )`: checks if a string is in a named set, returning a `bool` + +### Moderation Effects (Actions) + +"Flags" are a concept invented for automod. They are essentially private labels: string values attached to a subject (account or record) and persisted. + +Rules can take account-level actions using the following methods: + +- `c.AddAccountFlag(val string)` +- `c.AddAccountLabel(val string)` +- `c.ReportAccount(reason string, comment string)` +- `c.TakedownAccount()` + +The `RecordContext` additionally has record-level equivalents for all these methods. + +### Other Stuff + +- `c.Logger`: a `log/slog` logging interface. Logging currently happens immediately, instead of being accumulated as an "effect" +- `c.Directory()`: returns an `identity.Directory` (interface), which can be used for (cached) identity resolution + +## Development Process + +When deploying a new rule, it is recommended to start with a minimal action, like setting a flag or just logging. Any "action" (including new flag creation) can result in a Slack notification. You can gain confidence in the rule by running against the full firehose with these limited actions, tweaking the rule until it seems to have acceptable sensitivity (eg, few false positives), and then escalate the actions to reporting (adds to the human review queue), or action-and-report (label or takedown, and concurrently report for humans to review the action). + +### Network Data + +The `hepa` command provides `process-record` and `process-recent` sub-commands which will pull an existing individual record (by AT-URI) or all recent bsky posts for an account (by handle or DID), which can be helpful for testing. + +There is also a `capture-recent` sub-command which will save a snapshot ("capture") of the current account identity and profile, and recent bsky posts, as JSON. This can be combined with testing helpers (which will load the capture and push it through a mock rules engine) to test that new rules actually trigger as expected against real-world data. + +Note that, of course, any real-world captures should have identifying or otherwise sensitive information redacted or replaced before committing to git. + + +## Examples + +Here is a trivial post record rule: + +```golang +// the GTUBE string is a special value historically used to test email spam filtering behavior +var gtubeString = "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" + +func GtubePostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if strings.Contains(post.Text, gtubeString) { + c.AddRecordLabel("spam") + } + return nil +} +``` + +Every new (or updated) post is checked for an exact string match, and is labeled "spam" if found. diff --git a/automod/README.md b/automod/README.md new file mode 100644 index 000000000..21ba475d1 --- /dev/null +++ b/automod/README.md @@ -0,0 +1,33 @@ +`indigo/automod`: rules engine for anti-spam and other moderation tasks +======================================================================= + +automod is a "rules engine" framework, used to augment human moderators in the atproto network by proactively identifying patterns of behavior and content. Batches of rules are processed for novel "events" such as a new post or update of an account handle. Counters and other statistics are collected, which can drive subsequent rule invocations. The outcome of rules can be moderation events like "report account for human review" or "label post". Much of what this framework does is simply aggregating and maintaining caches of relevant metadata about accounts and pieces of content, so that rules have efficient access to this information. + +A primary design goal is to have a flexible framework to allow new rules to be written and deployed rapidly in response to new patterns of spam and abuse. + +Some example rules are included in the `automod/rules` package, but the expectation is that some real-world rules will be kept secret. + +Code for subscribing to a firehose is not included here; see `../cmd/hepa` for a service daemon built on this package. + +API reference documentation can be found on [pkg.go.dev](https://pkg.go.dev/github.com/bluesky-social/indigo/automod). + +## Architecture + +The runtime (`automod.Engine`) manages network requests, caching, and configuration. Outside calling code makes concurrent calls to the `Process*` methods that the runtime provides. The runtime constructs event context structs (eg, `automod.RecordContext`), hydrates relevant metadata from (cached) external services, and then executes a configured set of rules on the event. Rules may request additional information (via the `Context` object), do arbitrary local compute, and then update the context with "effects" (such as moderation actions). After all rules have run, the runtime will inspect the context and persist any side-effects, such as updating counter state and pushing any new moderation actions to external services. + +The runtime maintains state in several "stores", each of which has an interface and both in-memory and Redis implementations. The automod stores are semi-ephemeral: they are persisted and are important state for rules to work as expected, but they are not a canonical or long-term store for moderation decisions or actions. It is expected that Redis is used in virtually all deployments. The store types are: + +- `automod/cachestore`: generic data caching with expiration (TTL) and explicit purging. Used to cache account-level metadata, including identity lookups and (if available) private account metadata +- `automod/countstore`: keyed integer counters with time bucketing (eg, "hour", "day", "total"). Also includes probabilistic "distinct value" counters (eg, Redis HyperLogLog counters, with roughly 2% precision) +- `automod/setstore`: configurable static string sets. May eventually be runtime configurable +- `automod/flagstore`: mechanism to keep track of automod-generated "flags" (like labels or hashtags) on accounts or records. Mostly used to detect *new* flags. May eventually be moved in to the moderation service itself, similar to labels + +## Prior Art + +* The [SQRL language](https://sqrl-lang.github.io/sqrl/) and runtime was originally developed by an industry vendor named Smyte, then acquired by Twitter, with some core Javascript components released open source in 2023. The SQRL documentation is extensive and describes many of the design trade-offs and features specific to rules engines. Bluesky considered adopting SQRL but decided to start with a simpler runtime with rules in a known language (golang). + +* Reddit's [automod system](https://www.reddit.com/wiki/automoderator/) is simple an accessible for non-technical sub-reddit community moderators. Discord has a large ecosystem of bots which can help communities manage some moderation tasks, in particular mitigating spam and brigading. + +* Facebook's FXL and Haxl rule languages have been in use for over a decade. The 2012 paper ["The Facebook Immune System"](https://css.csail.mit.edu/6.858/2012/readings/facebook-immune.pdf) gives a good overview of design goals and how a rules engine fits in to a an overall anti-spam/anti-abuse pipeline. + +* Email anti-spam systems like SpamAssassin and rspamd have been modular and configurable for several decades. diff --git a/automod/cachestore/cachestore.go b/automod/cachestore/cachestore.go new file mode 100644 index 000000000..593364403 --- /dev/null +++ b/automod/cachestore/cachestore.go @@ -0,0 +1,11 @@ +package cachestore + +import ( + "context" +) + +type CacheStore interface { + Get(ctx context.Context, name, key string) (string, error) + Set(ctx context.Context, name, key string, val string) error + Purge(ctx context.Context, name, key string) error +} diff --git a/automod/cachestore/cachestore_mem.go b/automod/cachestore/cachestore_mem.go new file mode 100644 index 000000000..3592554e3 --- /dev/null +++ b/automod/cachestore/cachestore_mem.go @@ -0,0 +1,36 @@ +package cachestore + +import ( + "context" + "time" + + "github.com/hashicorp/golang-lru/v2/expirable" +) + +type MemCacheStore struct { + Data *expirable.LRU[string, string] +} + +func NewMemCacheStore(capacity int, ttl time.Duration) MemCacheStore { + return MemCacheStore{ + Data: expirable.NewLRU[string, string](capacity, nil, ttl), + } +} + +func (s MemCacheStore) Get(ctx context.Context, name, key string) (string, error) { + v, ok := s.Data.Get(name + "/" + key) + if !ok { + return "", nil + } + return v, nil +} + +func (s MemCacheStore) Set(ctx context.Context, name, key string, val string) error { + s.Data.Add(name+"/"+key, val) + return nil +} + +func (s MemCacheStore) Purge(ctx context.Context, name, key string) error { + s.Data.Remove(name + "/" + key) + return nil +} diff --git a/automod/cachestore/cachestore_redis.go b/automod/cachestore/cachestore_redis.go new file mode 100644 index 000000000..3872dc445 --- /dev/null +++ b/automod/cachestore/cachestore_redis.go @@ -0,0 +1,71 @@ +package cachestore + +import ( + "context" + "time" + + "github.com/go-redis/cache/v9" + "github.com/redis/go-redis/v9" +) + +type RedisCacheStore struct { + Data *cache.Cache + TTL time.Duration +} + +var _ CacheStore = (*RedisCacheStore)(nil) + +func NewRedisCacheStore(redisURL string, ttl time.Duration) (*RedisCacheStore, error) { + ctx := context.Background() + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + rdb := redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(ctx).Result() + if err != nil { + return nil, err + } + data := cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(10_000, ttl), + }) + return &RedisCacheStore{ + Data: data, + TTL: ttl, + }, nil +} + +func redisCacheKey(name, key string) string { + return "cache/" + name + "/" + key +} + +func (s RedisCacheStore) Get(ctx context.Context, name, key string) (string, error) { + var val string + err := s.Data.Get(ctx, redisCacheKey(name, key), &val) + if err == cache.ErrCacheMiss { + return "", nil + } + if err != nil { + return "", err + } + return val, nil +} + +func (s RedisCacheStore) Set(ctx context.Context, name, key string, val string) error { + return s.Data.Set(&cache.Item{ + Ctx: ctx, + Key: redisCacheKey(name, key), + Value: val, + TTL: s.TTL, + }) +} + +func (s RedisCacheStore) Purge(ctx context.Context, name, key string) error { + err := s.Data.Delete(ctx, redisCacheKey(name, key)) + if err == cache.ErrCacheMiss { + return nil + } + return err +} diff --git a/automod/cachestore/doc.go b/automod/cachestore/doc.go new file mode 100644 index 000000000..e5c31ba15 --- /dev/null +++ b/automod/cachestore/doc.go @@ -0,0 +1,6 @@ +// Automod component for caching arbitrary data (as JSON strings) with a fixed TTL and purging. +// +// Includes an interface and implementations using redis and in-process memory. +// +// This is used by the rules engine to cache things like account metadata, improving latency and reducing load on authoritative backend systems. +package cachestore diff --git a/automod/capture/capture.go b/automod/capture/capture.go new file mode 100644 index 000000000..1747f9981 --- /dev/null +++ b/automod/capture/capture.go @@ -0,0 +1,43 @@ +package capture + +import ( + "context" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" +) + +type AccountCapture struct { + CapturedAt syntax.Datetime `json:"capturedAt"` + AccountMeta automod.AccountMeta `json:"accountMeta"` + PostRecords []comatproto.RepoListRecords_Record `json:"postRecords"` +} + +func CaptureRecent(ctx context.Context, eng *automod.Engine, atid syntax.AtIdentifier, limit int) (*AccountCapture, error) { + ident, records, err := FetchRecent(ctx, eng, atid, limit) + if err != nil { + return nil, err + } + pr := []comatproto.RepoListRecords_Record{} + for _, r := range records { + if r != nil { + pr = append(pr, *r) + } + } + + am, err := eng.GetAccountMeta(ctx, ident) + if err != nil { + return nil, err + } + + // auto-clear sensitive PII (eg, account email) + am.Private = nil + + ac := AccountCapture{ + CapturedAt: syntax.DatetimeNow(), + AccountMeta: *am, + PostRecords: pr, + } + return &ac, nil +} diff --git a/automod/capture/capture_test.go b/automod/capture/capture_test.go new file mode 100644 index 000000000..e1fc3c2ef --- /dev/null +++ b/automod/capture/capture_test.go @@ -0,0 +1,25 @@ +package capture + +import ( + "context" + "testing" + + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/engine" + "github.com/stretchr/testify/assert" +) + +func TestNoOpCaptureReplyRule(t *testing.T) { + ctx := context.Background() + assert := assert.New(t) + + eng := engine.EngineTestFixture() + capture := MustLoadCapture("testdata/capture_atprotocom.json") + assert.NoError(ProcessCaptureRules(&eng, capture)) + c, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay) + assert.NoError(err) + assert.Equal(0, c) + c, err = eng.Counters.GetCount(ctx, "automod-quota", "takedown", countstore.PeriodDay) + assert.NoError(err) + assert.Equal(0, c) +} diff --git a/automod/capture/doc.go b/automod/capture/doc.go new file mode 100644 index 000000000..514496975 --- /dev/null +++ b/automod/capture/doc.go @@ -0,0 +1,4 @@ +// Automod development helpers for fetching and saving snapshots of real-world content and metadata. +// +// NOTE: real-world content is sensitive, even if it is abusive. +package capture diff --git a/automod/capture/fetch.go b/automod/capture/fetch.go new file mode 100644 index 000000000..1139eaaa0 --- /dev/null +++ b/automod/capture/fetch.go @@ -0,0 +1,107 @@ +package capture + +import ( + "bytes" + "context" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/xrpc" +) + +func FetchAndProcessRecord(ctx context.Context, eng *automod.Engine, aturi syntax.ATURI) error { + // resolve URI, identity, and record + if aturi.RecordKey() == "" { + return fmt.Errorf("need a full, not partial, AT-URI: %s", aturi) + } + ident, err := eng.Directory.Lookup(ctx, aturi.Authority()) + if err != nil { + return fmt.Errorf("resolving AT-URI authority: %v", err) + } + pdsURL := ident.PDSEndpoint() + if pdsURL == "" { + return fmt.Errorf("could not resolve PDS endpoint for AT-URI account: %s", ident.DID.String()) + } + pdsClient := xrpc.Client{Host: pdsURL} + + eng.Logger.Info("fetching record", "did", ident.DID.String(), "collection", aturi.Collection().String(), "rkey", aturi.RecordKey().String()) + out, err := comatproto.RepoGetRecord(ctx, &pdsClient, "", aturi.Collection().String(), ident.DID.String(), aturi.RecordKey().String()) + if err != nil { + return fmt.Errorf("fetching record from PDS (%s): %v", aturi, err) + } + if out.Cid == nil { + return fmt.Errorf("expected a CID in getRecord response") + } + recCID := syntax.CID(*out.Cid) + recBuf := new(bytes.Buffer) + if err := out.Value.Val.MarshalCBOR(recBuf); err != nil { + return err + } + recBytes := recBuf.Bytes() + op := automod.RecordOp{ + Action: automod.CreateOp, + DID: ident.DID, + Collection: aturi.Collection(), + RecordKey: aturi.RecordKey(), + CID: &recCID, + RecordCBOR: recBytes, + } + return eng.ProcessRecordOp(ctx, op) +} + +func FetchRecent(ctx context.Context, eng *automod.Engine, atid syntax.AtIdentifier, limit int) (*identity.Identity, []*comatproto.RepoListRecords_Record, error) { + ident, err := eng.Directory.Lookup(ctx, atid) + if err != nil { + return nil, nil, fmt.Errorf("failed to resolve AT identifier: %v", err) + } + pdsURL := ident.PDSEndpoint() + if pdsURL == "" { + return nil, nil, fmt.Errorf("could not resolve PDS endpoint for account: %s", ident.DID.String()) + } + pdsClient := xrpc.Client{Host: pdsURL} + + resp, err := comatproto.RepoListRecords(ctx, &pdsClient, "app.bsky.feed.post", "", int64(limit), ident.DID.String(), false) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch record list: %v", err) + } + eng.Logger.Info("got recent posts", "did", ident.DID.String(), "pds", pdsURL, "count", len(resp.Records)) + return ident, resp.Records, nil +} + +func FetchAndProcessRecent(ctx context.Context, eng *automod.Engine, atid syntax.AtIdentifier, limit int) error { + + ident, records, err := FetchRecent(ctx, eng, atid, limit) + if err != nil { + return err + } + // records are most-recent first; we want recent but oldest-first, so iterate backwards + for i := range records { + rec := records[len(records)-i-1] + aturi, err := syntax.ParseATURI(rec.Uri) + if err != nil { + return fmt.Errorf("parsing PDS record response: %v", err) + } + recCID := syntax.CID(rec.Cid) + recBuf := new(bytes.Buffer) + if err := rec.Value.Val.MarshalCBOR(recBuf); err != nil { + return err + } + recBytes := recBuf.Bytes() + op := automod.RecordOp{ + Action: automod.CreateOp, + DID: ident.DID, + Collection: aturi.Collection(), + RecordKey: aturi.RecordKey(), + CID: &recCID, + RecordCBOR: recBytes, + } + err = eng.ProcessRecordOp(ctx, op) + if err != nil { + return err + } + } + return nil +} diff --git a/automod/capture/testdata/capture_atprotocom.json b/automod/capture/testdata/capture_atprotocom.json new file mode 100644 index 000000000..b39e7e05e --- /dev/null +++ b/automod/capture/testdata/capture_atprotocom.json @@ -0,0 +1,823 @@ +{ + "capturedAt": "2023-12-09T05:45:35.662Z", + "accountMeta": { + "Identity": { + "DID": "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + "Handle": "atproto.com", + "AlsoKnownAs": [ + "at://atproto.com" + ], + "Services": { + "atproto_pds": { + "Type": "AtprotoPersonalDataServer", + "URL": "https://enoki.us-east.host.bsky.network" + } + }, + "Keys": { + "atproto": { + "Type": "Multikey", + "PublicKeyMultibase": "zQ3shunBKsXixLxKtC5qeSG9E4J5RkGN57im31pcTzbNQnm5w" + } + } + }, + "Profile": { + "HasAvatar": true, + "Description": "Social networking technology created by Bluesky. \n\nDeveloper-focused account. Follow @bsky.app for general announcements!\n\nDocs: atproto.com\nCommunity: atproto.com/community\nDeveloper blog: https://atproto.com/blog", + "DisplayName": "AT Protocol Developers" + }, + "Private": null, + "AccountLabels": null, + "AccountNegatedLabels": null, + "AccountFlags": [], + "FollowersCount": 47411, + "FollowsCount": 16, + "PostsCount": 86, + "Takendown": false + }, + "postRecords": [ + { + "cid": "bafyreih22okgdungwln2lwqt2kd4dd2ynzx3fyy3yefh5koocpizlngqfq", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kftldxbvq22p", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-06T00:21:07.723Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "We’re preparing to launch a public web interface soon, and we’re excited for more people to see your custom feeds! Ahead of that, we want to make sure that your feeds will work smoothly to handle ...", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreicgt2g7a3dtocucsdzfka2vbtqa56yll2bmpge2gk7f26ry4nn7ky" + }, + "mimeType": "image/jpeg", + "size": 350013 + }, + "title": "Expected feed generator behavior for the public web view · bluesky-social/atproto · Discussion #19...", + "uri": "https://github.com/bluesky-social/atproto/discussions/1933" + } + }, + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreiheaai2dcnd676fwxbpoozzu7rzin4gwswls2bob6noi24t5okxcq", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kftlbujmfk24" + }, + "root": { + "cid": "bafyreiheaai2dcnd676fwxbpoozzu7rzin4gwswls2bob6noi24t5okxcq", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kftlbujmfk24" + } + }, + "text": "[for developers]\n\ntl;dr if your feed requires auth (e.g. likes, mutuals), return a 401 with an optional custom message. \n\nif your feed is experiencing too much traffic, return a 429.\n\nwe'll have fallbacks too in case a influx of traffic is making it difficult on your end 🙏 \n\nmore details:" + } + }, + { + "cid": "bafyreiheaai2dcnd676fwxbpoozzu7rzin4gwswls2bob6noi24t5okxcq", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kftlbujmfk24", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-06T00:19:57.703Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "We’re preparing to launch a public web interface soon, and we’re excited for more people to see your custom feeds! Ahead of that, we want to make sure that your feeds will work smoothly to handle ...", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreicgt2g7a3dtocucsdzfka2vbtqa56yll2bmpge2gk7f26ry4nn7ky" + }, + "mimeType": "image/jpeg", + "size": 350013 + }, + "title": "Expected feed generator behavior for the public web view · bluesky-social/atproto · Discussion #19...", + "uri": "https://github.com/bluesky-social/atproto/discussions/1933" + } + }, + "langs": [ + "en" + ], + "text": "hey developers, we're excited for more people to see your custom feeds soon when we launch the public web interface! \n\nahead of that, we want to ensure that your feeds will work smoothly to handle both logged-out users and a potential increase in traffic. read some details on expected behavior here:" + } + }, + { + "cid": "bafyreieyg5pi2anhktagcnlv5cuwcbumwpx2fko624pkk6fqskmhsq426a", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqkmk5kpv2o", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T19:30:03.024Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub. - GitHub - snarfed/bridgy-fed: 🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreidplhjcnrl2c74r3xs7nh7k7q3ny6ul7cgxr2fophblvdeky6t64e" + }, + "mimeType": "image/jpeg", + "size": 347998 + }, + "title": "GitHub - snarfed/bridgy-fed: 🕸 Bridges the IndieWeb to Mastodon and the fediverse via ActivityPub...", + "uri": "https://github.com/snarfed/bridgy-fed" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://github.com/snarfed/bridgy-fed" + } + ], + "index": { + "byteEnd": 92, + "byteStart": 66 + } + }, + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:fdme4gb7mu7zrie7peay7tst" + } + ], + "index": { + "byteEnd": 149, + "byteStart": 137 + } + } + ], + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreifaidyl62p4snkdwsygviemsxyidi3cd7dxvjomh5644sovxhsppa", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqklhpalh2c" + }, + "root": { + "cid": "bafyreibiimdwmsp5mqpm7utqcdmvo6fdqmofblp5obs3h7ub6652zyooci", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqkkjdkic2e" + } + }, + "text": "Bridgy Fed is an open-source project — check out the code here: github.com/snarfed/brid...\n\nStay updated with the project by following @snarfed.org!" + } + }, + { + "cid": "bafyreifaidyl62p4snkdwsygviemsxyidi3cd7dxvjomh5644sovxhsppa", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqklhpalh2c", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T19:29:26.790Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "Bridgy Fed is a bridge between decentralized social networks that already has initial Bluesky support. The project will launch publicly when Bluesky launches federation early next year.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreibjcz2ahvy6iwx2ofgmnbl7xevwo2k3e36op6lngvpnlgf6yyqbfq" + }, + "mimeType": "image/jpeg", + "size": 736017 + }, + "title": "Featured Community Project: Bridgy Fed | AT Protocol", + "uri": "https://atproto.com/blog/feature-bridgyfed" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://atproto.com/blog/feature-bridgyfed" + } + ], + "index": { + "byteEnd": 265, + "byteStart": 238 + } + } + ], + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreibiimdwmsp5mqpm7utqcdmvo6fdqmofblp5obs3h7ub6652zyooci", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqkkjdkic2e" + }, + "root": { + "cid": "bafyreibiimdwmsp5mqpm7utqcdmvo6fdqmofblp5obs3h7ub6652zyooci", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqkkjdkic2e" + } + }, + "text": "Bridgy Fed is a third-party tool that allows you to follow anyone on any other network, see their posts, reply or like or repost them, and those interactions flow across to their network and vice versa.\n\nRead more about the project here: atproto.com/blog/feature..." + } + }, + { + "cid": "bafyreibiimdwmsp5mqpm7utqcdmvo6fdqmofblp5obs3h7ub6652zyooci", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfqkkjdkic2e", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T19:28:55.054Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "Bridgy Fed is a bridge between decentralized social networks that already has initial Bluesky support. The project will launch publicly when Bluesky launches federation early next year.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreibjcz2ahvy6iwx2ofgmnbl7xevwo2k3e36op6lngvpnlgf6yyqbfq" + }, + "mimeType": "image/jpeg", + "size": 736017 + }, + "title": "Featured Community Project: Bridgy Fed | AT Protocol", + "uri": "https://atproto.com/blog/feature-bridgyfed" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:fdme4gb7mu7zrie7peay7tst" + } + ], + "index": { + "byteEnd": 75, + "byteStart": 63 + } + }, + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://atproto.com/blog/feature-bridgyfed" + } + ], + "index": { + "byteEnd": 259, + "byteStart": 232 + } + } + ], + "langs": [ + "en" + ], + "text": "Check out the latest featured community project: Bridgy Fed by @snarfed.org!\n\nBridgy Fed is a bridge between social networks that currently supports the IndieWeb and the Fediverse, with full Bluesky support coming soon.\n\nRead more: atproto.com/blog/feature..." + } + }, + { + "cid": "bafyreidm5mpgmcnsyjmfnbpmi3m3n4673pp72ol2j4c5q7f6vobqwhqhx4", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfdph62zey2f", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-29T16:51:54.613Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "Open Core Summit is the world's largest community of COSS (commercial open source software) company builders, founders, investors and more.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreifkvhuqfe5wuggwl4prdu7hojthfxue5dzqdselxrpgur5m3rbcdm" + }, + "mimeType": "image/jpeg", + "size": 625220 + }, + "title": "Open Core Summit", + "uri": "https://opencoresummit.com/" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://opencoresummit.com/" + } + ], + "index": { + "byteEnd": 241, + "byteStart": 223 + } + } + ], + "langs": [ + "en" + ], + "text": "Developers, we'll be at Open Core Summit on Dec 6-7 in San Francisco! Stop by to say hi and pick up some swag. 👾\n\nHere's a discount code generated by Open Core Summit for FOSS builders to meet us there: opensourcefrens\n\nopencoresummit.com" + } + }, + { + "cid": "bafyreidt2xwgu66lrwoob2zlz6uuppjbv2rl7y5wh7c2akrvj32epuvxw4", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kfdp7rlxtg27", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-29T16:47:46.702Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "Get started with the atproto API.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreieya7iitpu4okjtm7iexiwikj7t63ttlthad32ojsvjqhqbc3iwmi" + }, + "mimeType": "image/jpeg", + "size": 518394 + }, + "title": "TypeScript Quick Start Guide | AT Protocol", + "uri": "https://atproto.com/community/quick-start" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://atproto.com/community/quick-start" + } + ], + "index": { + "byteEnd": 32, + "byteStart": 5 + } + } + ], + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreibdypeqidcengvmqb7tisq46qmnaglkno4mozw4ktdyosmouzz2iy", + "uri": "at://did:plc:tp4brp6abnra3rvggvazeziq/app.bsky.feed.post/3kfdovnudaz2g" + }, + "root": { + "cid": "bafyreibdypeqidcengvmqb7tisq46qmnaglkno4mozw4ktdyosmouzz2iy", + "uri": "at://did:plc:tp4brp6abnra3rvggvazeziq/app.bsky.feed.post/3kfdovnudaz2g" + } + }, + "text": "yep! atproto.com/community/qu..." + } + }, + { + "cid": "bafyreihac22tpufnn3rngrhgpixha7ffbbl27z4fny7owjejtjeqp37zxa", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kdrlsxkryo2f", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-09T18:33:48.645Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreiefdhpdngbr3rbagqik6lubuwmc7ttripb3ugttvvd26zcnkmwaxa", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kdrlslade72m" + }, + "root": { + "cid": "bafyreiefdhpdngbr3rbagqik6lubuwmc7ttripb3ugttvvd26zcnkmwaxa", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kdrlslade72m" + } + }, + "text": "We explicitly built Bluesky and the AT Protocol with a simple and straightforward user experience in mind. 💪 \n\nThe technical updates here are a behind-the-scenes look into the plumbing, which is useful for devs who want to follow along and are building their own projects on this tech." + } + }, + { + "cid": "bafyreiefdhpdngbr3rbagqik6lubuwmc7ttripb3ugttvvd26zcnkmwaxa", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kdrlslade72m", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-09T18:33:35.718Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:z72i7hdynmk6r22z27h6tvur" + } + ], + "index": { + "byteEnd": 218, + "byteStart": 209 + } + } + ], + "langs": [ + "en" + ], + "text": "Reminder: This is a developer-focused account, so if you see us using technical speak here, that's why!\n\nWe'll always make relevant info more accessible to all users, and if you haven't already, please follow @bsky.app for general announcements. 😊" + } + }, + { + "cid": "bafyreiai3kukjoxw5s5amfq24eirqbv5zofsfmfw5zon7z5yfmbu57y3ge", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kdow6fj6la2l", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-08T17:01:10.489Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "As part of our operational work in preparation for federation, we are starting to migrate PDS accounts away from the existing monolithic bsky.social instance to several *.*.host.bsky.network PDS in...", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreifphxg3k6bp3bzaw7b4qswm2hkewlqedkbl57rtcfq2d2c6bqeflu" + }, + "mimeType": "image/jpeg", + "size": 382219 + }, + "title": "Migrating bsky.social to Multiple PDS Instances (Nov 2023) · bluesky-social/atproto · Discussion #...", + "uri": "https://github.com/bluesky-social/atproto/discussions/1832" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://github.com/bluesky-social/atproto/discussions/1832" + } + ], + "index": { + "byteEnd": 195, + "byteStart": 169 + } + } + ], + "langs": [ + "en" + ], + "text": "Devs: you might've already seen some chatter about migrating accounts to servers named after mushrooms... \n\nRead the technical details and how it might affect you here: github.com/bluesky-soci..." + } + }, + { + "cid": "bafyreibs3iuez2urcyfwmwcceg5gilnvfz3svah7q4m5zd7rpiueodl5ra", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kdk7yc3pbz22", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-06T20:13:24.422Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "How to download and parse repository data exports using Go.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreiffvyhd4vqp7hef2zii6bs54zi4sw4z7g2zt5vx4j4ct5hrhewdyq" + }, + "mimeType": "image/jpeg", + "size": 599519 + }, + "title": "Download and Parse Repository Exports | AT Protocol", + "uri": "https://atproto.com/blog/repo-export" + } + }, + "langs": [ + "en" + ], + "text": "📝Just published a new dev blog post: \n\nOne of the core principles of atproto is simple access to public data. A user’s data is stored in a repository, which can be efficiently exported all together as a CAR file (.car). \n\nThis post describes how to export and parse a data repository." + } + }, + { + "cid": "bafyreiak2l4jmurdrhgaioo6owhbttgoyapxpg45afl4cgnlx2rhv4v2eq", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kd5uzahw5t2u", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-01T22:25:08.177Z", + "embed": { + "$type": "app.bsky.embed.record", + "record": { + "cid": "bafyreig6e5eynvb5bq7v4zpv7aiq6zmjz4gj2lhgr3kwkkbkriiofqlbmi", + "uri": "at://did:plc:wqowuobffl66jv3kpsvo7ak4/app.bsky.feed.post/3kd5tr4uex22v" + } + }, + "langs": [ + "en" + ], + "text": "do you subscribe to the For You custom feed and have opinions to share?\n\nthe third-party devs behind the feed are hosting a zoom for the next 4 hours to hear your thoughts! ⬇️" + } + }, + { + "cid": "bafyreihrrsdswqqco2mn3bhm3d7wdtioaqz7gr3tj7ms6cvakz2obv3qha", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kblbjgnevy25", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-10-12T19:23:09.439Z", + "embed": { + "$type": "app.bsky.embed.record", + "record": { + "cid": "bafyreialpvpg4w32go46d4dp2da5pm5ypamml4bqymegizogh6tegkqxmu", + "uri": "at://did:plc:p2cp5gopk7mgjegy6wadk3ep/app.bsky.feed.post/3kbl67fqu4k2t" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:sq6aa2wa32tiiqrbub64vcja" + } + ], + "index": { + "byteEnd": 24, + "byteStart": 12 + } + }, + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://github.com/mozzius/graysky" + } + ], + "index": { + "byteEnd": 132, + "byteStart": 106 + } + }, + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://atproto.com/community/projects#clients" + } + ], + "index": { + "byteEnd": 297, + "byteStart": 270 + } + } + ], + "langs": [ + "en" + ], + "text": "congrats to @graysky.app, a third-party open-source bluesky client! 🔥\n\ncheck out the source code here: github.com/mozzius/gray...\n\nbecause the AT Protocol is open, third-party developers can create entirely separate frontends for the network. find more clients here: atproto.com/community/pr..." + } + }, + { + "cid": "bafyreigc2hx7giirtsrotyeadf2tvmshonfcnwqtl7toom5te6rfcdgvgq", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbj5lqxjs42w", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-10-11T23:07:33.183Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "This post lays out the current AT Protocol development plan through the end of 2023", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreieya7iitpu4okjtm7iexiwikj7t63ttlthad32ojsvjqhqbc3iwmi" + }, + "mimeType": "image/jpeg", + "size": 518394 + }, + "title": "2023 Protocol Roadmap | AT Protocol", + "uri": "https://atproto.com/blog/2023-protocol-roadmap" + } + }, + "langs": [ + "en" + ], + "text": "2023 Protocol Roadmap — just published on the dev blog! \n\nThis blog is written for developers already familiar with atproto concepts and terminology. We are pushing towards federation on the production network early next year (2024), if development continues as planned." + } + }, + { + "cid": "bafyreigofn3o6fib6bmwlxlmj7kqudusrso4ghiy3oymx3jpv5ifr7prim", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbj5ighpaw27", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-10-11T23:05:41.516Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "A collection of example projects and scripts for atproto development. - GitHub - bluesky-social/cookbook: A collection of example projects and scripts for atproto development.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreihamtttuw7j5tjj74l44h7xms6gf5ywaevysn7tktykrob2aurlmi" + }, + "mimeType": "image/jpeg", + "size": 311389 + }, + "title": "GitHub - bluesky-social/cookbook: A collection of example projects and scripts for atproto developme...", + "uri": "https://github.com/bluesky-social/cookbook" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://github.com/bluesky-social/cookbook" + } + ], + "index": { + "byteEnd": 242, + "byteStart": 216 + } + } + ], + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreigviq2nvak7ebhh7tyoi527k6d5li4ck7txxawckayf4wq47jg2y4", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbj5haaqar2r" + }, + "root": { + "cid": "bafyreigviq2nvak7ebhh7tyoi527k6d5li4ck7txxawckayf4wq47jg2y4", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbj5haaqar2r" + } + }, + "text": "We also have a new \"cookbook\" repo now that will be a home for example code snippets and starter kits. \n\nCurrently, you can find:\n• how to make a Bluesky post in Python\n• how to make a Bluesky bot in TypeScript\n\ngithub.com/bluesky-soci..." + } + }, + { + "cid": "bafyreigviq2nvak7ebhh7tyoi527k6d5li4ck7txxawckayf4wq47jg2y4", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbj5haaqar2r", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-10-11T23:05:01.389Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "Pointers to what you can already build on atproto, and what you can expect soon.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreieya7iitpu4okjtm7iexiwikj7t63ttlthad32ojsvjqhqbc3iwmi" + }, + "mimeType": "image/jpeg", + "size": 518394 + }, + "title": "Building on the AT Protocol | AT Protocol", + "uri": "https://atproto.com/blog/building-on-atproto" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://atproto.com/blog/building-on-atproto" + } + ], + "index": { + "byteEnd": 217, + "byteStart": 190 + } + } + ], + "langs": [ + "en" + ], + "text": "Are you a developer looking for a way to get started with atproto dev? 👀\n\nRead our latest blog post with some pointers to what you can already built on the protocol, with starter code! \n\natproto.com/blog/buildin..." + } + }, + { + "cid": "bafyreifitblocnnwzlhmcftromkauc3lprs37gvyjaewbfbp3d6hahybo4", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbg4ksc3in2n", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-10-10T18:11:08.161Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://status.skyfeed.xyz/" + } + ], + "index": { + "byteEnd": 166, + "byteStart": 148 + } + } + ], + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreihtpwbcoqpgq5iknqsmgo4jq6le3odkbzospye3q5jtgzj7q5oepa", + "uri": "at://did:plc:cipv3arp27zug6ugyk7fv76o/app.bsky.feed.post/3kbg45b6drf2s" + }, + "root": { + "cid": "bafyreiduoaidw46xhv4kyffxizusembgrtuyse7yloenkjwk2ylc3jatyy", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbg3t7ypo52j" + } + }, + "text": "don't think so: DIDs themselves are not changing, and keys aren't changing, only the way public keys are encoded.\n\nyou can check skyfeed status at: status.skyfeed.xyz" + } + }, + { + "cid": "bafyreiduoaidw46xhv4kyffxizusembgrtuyse7yloenkjwk2ylc3jatyy", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbg3t7ypo52j", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-10-10T17:57:57.125Z", + "embed": { + "$type": "app.bsky.embed.record", + "record": { + "cid": "bafyreifzavcz43kc753s2hjwstzangmx7cce3cjpiaesdnt6ssxyxwm5ny", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbdraubs7l2f" + } + }, + "langs": [ + "en" + ], + "text": "this change is rolling out to 'plc.directory' now!" + } + }, + { + "cid": "bafyreifzavcz43kc753s2hjwstzangmx7cce3cjpiaesdnt6ssxyxwm5ny", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kbdraubs7l2f", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-10-09T19:43:24.015Z", + "embed": { + "$type": "app.bsky.embed.record", + "record": { + "cid": "bafyreibwh5d74t4xsuxjhg3sy4f2mlybcwtra53wfzst7foejhdv5c73zi", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kb4ce57t6d2h" + } + }, + "langs": [ + "en" + ], + "text": "reminder for developers: this DID document syntax change will be deployed tomorrow on the main 'plc.directory' server! \n\nfor most devs, this shouldn't require any/specific action on your end." + } + }, + { + "cid": "bafyreibwh5d74t4xsuxjhg3sy4f2mlybcwtra53wfzst7foejhdv5c73zi", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kb4ce57t6d2h", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-10-06T20:28:09.688Z", + "embed": { + "$type": "app.bsky.embed.external", + "external": { + "description": "The Bluesky AppView now consumes from a Bluesky BGS, instead of directly from the PDS. Also, we'll be updating the DID document public key syntax.", + "thumb": { + "$type": "blob", + "ref": { + "$link": "bafkreieya7iitpu4okjtm7iexiwikj7t63ttlthad32ojsvjqhqbc3iwmi" + }, + "mimeType": "image/jpeg", + "size": 518394 + }, + "title": "Bluesky BGS and DID Document Formatting Changes | AT Protocol", + "uri": "https://atproto.com/blog/bgs-and-did-doc" + } + }, + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://plc.directory" + } + ], + "index": { + "byteEnd": 182, + "byteStart": 169 + } + }, + { + "features": [ + { + "$type": "app.bsky.richtext.facet#link", + "uri": "https://atproto.com/blog/bgs-and-did-doc" + } + ], + "index": { + "byteEnd": 279, + "byteStart": 252 + } + } + ], + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreifovukaaagkr2n5bj4oqsqwswhd6ulqdfqfi7zdczgjiab2l3u5oa", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kb4ccsn2nq25" + }, + "root": { + "cid": "bafyreiedct7s34kqdugycvv3juaujiz4tjwfpitxsqhbma7r6hqg2wigfq", + "uri": "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.feed.post/3kb4cbhzw742d" + } + }, + "text": "• We also want to remind folks that we are planning to update the DID document public key syntax to “Multikey” format next week on the main network PLC directory (plc.directory). These changes are live now on the sandbox PLC directory.\n\nDetails: atproto.com/blog/bgs-and..." + } + } + ] +} diff --git a/automod/capture/testing.go b/automod/capture/testing.go new file mode 100644 index 000000000..fbe00d6cb --- /dev/null +++ b/automod/capture/testing.go @@ -0,0 +1,84 @@ +package capture + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" +) + +func MustLoadCapture(capPath string) AccountCapture { + f, err := os.Open(capPath) + if err != nil { + panic(err) + } + defer func() { _ = f.Close() }() + + raw, err := io.ReadAll(f) + if err != nil { + panic(err) + } + + var capture AccountCapture + if err := json.Unmarshal(raw, &capture); err != nil { + panic(err) + } + return capture +} + +// Test helper which processes all the records from a capture. Intentionally exported, for use in other packages. +// +// This method replaces any pre-existing directory on the engine with a mock directory. +func ProcessCaptureRules(eng *automod.Engine, capture AccountCapture) error { + ctx := context.Background() + + did := capture.AccountMeta.Identity.DID + handle := capture.AccountMeta.Identity.Handle.String() + dir := identity.NewMockDirectory() + dir.Insert(*capture.AccountMeta.Identity) + eng.Directory = &dir + + // initial identity rules + identEvent := comatproto.SyncSubscribeRepos_Identity{ + Did: did.String(), + Handle: &handle, + Seq: 12345, + Time: syntax.DatetimeNow().String(), + } + eng.ProcessIdentityEvent(ctx, identEvent) + + // all the post rules + for _, pr := range capture.PostRecords { + aturi, err := syntax.ParseATURI(pr.Uri) + if err != nil { + return err + } + did, err := aturi.Authority().AsDID() + if err != nil { + return err + } + recCID := syntax.CID(pr.Cid) + recBuf := new(bytes.Buffer) + if err := pr.Value.Val.MarshalCBOR(recBuf); err != nil { + return err + } + recBytes := recBuf.Bytes() + eng.Logger.Debug("processing record", "did", did) + op := automod.RecordOp{ + Action: automod.CreateOp, + DID: did, + Collection: aturi.Collection(), + RecordKey: aturi.RecordKey(), + CID: &recCID, + RecordCBOR: recBytes, + } + eng.ProcessRecordOp(ctx, op) + } + return nil +} diff --git a/automod/consumer/doc.go b/automod/consumer/doc.go new file mode 100644 index 000000000..fa8ccdcb2 --- /dev/null +++ b/automod/consumer/doc.go @@ -0,0 +1,2 @@ +// Code for consuming from atproto firehose and ozone event stream, pushing events in to automod engine. +package consumer diff --git a/automod/consumer/firehose.go b/automod/consumer/firehose.go new file mode 100644 index 000000000..ac0d1c821 --- /dev/null +++ b/automod/consumer/firehose.go @@ -0,0 +1,273 @@ +package consumer + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "net/http" + "net/url" + "sync/atomic" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/schedulers/autoscaling" + "github.com/bluesky-social/indigo/events/schedulers/parallel" + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/repo" + "github.com/bluesky-social/indigo/repomgr" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/gorilla/websocket" + "github.com/redis/go-redis/v9" +) + +// TODO: should probably make this not hepa-specific; or even configurable +var firehoseCursorKey = "hepa/seq" + +type FirehoseConsumer struct { + Parallelism int + Logger *slog.Logger + RedisClient *redis.Client + Engine *automod.Engine + Host string + + // TODO: prefilter record collections; or predicate function? + // TODO: enable/disable event types; or predicate function? + + // lastSeq is the most recent event sequence number we've received and begun to handle. + // This number is periodically persisted to redis, if redis is present. + // The value is best-effort (the stream handling itself is concurrent, so event numbers may not be monotonic), + // but nonetheless, you must use atomics when updating or reading this (to avoid data races). + lastSeq int64 +} + +func (fc *FirehoseConsumer) Run(ctx context.Context) error { + + if fc.Engine == nil { + return fmt.Errorf("nil engine") + } + + cur, err := fc.ReadLastCursor(ctx) + if err != nil { + return err + } + + dialer := websocket.DefaultDialer + u, err := url.Parse(fc.Host) + if err != nil { + return fmt.Errorf("invalid Host URI: %w", err) + } + u.Path = "xrpc/com.atproto.sync.subscribeRepos" + if cur != 0 { + u.RawQuery = fmt.Sprintf("cursor=%d", cur) + } + fc.Logger.Info("subscribing to repo event stream", "upstream", fc.Host, "cursor", cur) + con, _, err := dialer.Dial(u.String(), http.Header{ + "User-Agent": []string{fmt.Sprintf("hepa/%s", versioninfo.Short())}, + }) + if err != nil { + return fmt.Errorf("subscribing to firehose failed (dialing): %w", err) + } + + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + atomic.StoreInt64(&fc.lastSeq, evt.Seq) + return fc.HandleRepoCommit(ctx, evt) + }, + RepoIdentity: func(evt *comatproto.SyncSubscribeRepos_Identity) error { + atomic.StoreInt64(&fc.lastSeq, evt.Seq) + if err := fc.Engine.ProcessIdentityEvent(context.Background(), *evt); err != nil { + fc.Logger.Error("processing repo identity failed", "did", evt.Did, "seq", evt.Seq, "err", err) + } + return nil + }, + RepoAccount: func(evt *comatproto.SyncSubscribeRepos_Account) error { + atomic.StoreInt64(&fc.lastSeq, evt.Seq) + if err := fc.Engine.ProcessAccountEvent(context.Background(), *evt); err != nil { + fc.Logger.Error("processing repo account failed", "did", evt.Did, "seq", evt.Seq, "err", err) + } + return nil + }, + // NOTE: no longer process #handle events + // NOTE: no longer process #tombstone events + } + + var scheduler events.Scheduler + if fc.Parallelism > 0 { + // use a fixed-parallelism scheduler if configured + scheduler = parallel.NewScheduler( + fc.Parallelism, + 1000, + fc.Host, + rsc.EventHandler, + ) + fc.Logger.Info("hepa scheduler configured", "scheduler", "parallel", "initial", fc.Parallelism) + } else { + // otherwise use auto-scaling scheduler + scaleSettings := autoscaling.DefaultAutoscaleSettings() + // start at higher parallelism (somewhat arbitrary) + scaleSettings.Concurrency = 4 + scaleSettings.MaxConcurrency = 200 + scheduler = autoscaling.NewScheduler(scaleSettings, fc.Host, rsc.EventHandler) + fc.Logger.Info("hepa scheduler configured", "scheduler", "autoscaling", "initial", scaleSettings.Concurrency, "max", scaleSettings.MaxConcurrency) + } + + return events.HandleRepoStream(ctx, con, scheduler, fc.Logger) +} + +// NOTE: for now, this function basically never errors, just logs and returns nil. Should think through error processing better. +func (fc *FirehoseConsumer) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) error { + + logger := fc.Logger.With("event", "commit", "did", evt.Repo, "rev", evt.Rev, "seq", evt.Seq) + logger.Debug("received commit event") + + if evt.TooBig { + logger.Warn("skipping tooBig events for now") + return nil + } + + did, err := syntax.ParseDID(evt.Repo) + if err != nil { + logger.Error("bad DID syntax in event", "err", err) + return nil + } + + rr, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks)) + if err != nil { + logger.Error("failed to read repo from car", "err", err) + return nil + } + + for _, op := range evt.Ops { + logger = logger.With("eventKind", op.Action, "path", op.Path) + collection, rkey, err := syntax.ParseRepoPath(op.Path) + if err != nil { + logger.Error("invalid path in repo op", "err", err) + return nil + } + + ek := repomgr.EventKind(op.Action) + switch ek { + case repomgr.EvtKindCreateRecord, repomgr.EvtKindUpdateRecord: + // read the record bytes from blocks, and verify CID + rc, recCBOR, err := rr.GetRecordBytes(ctx, op.Path) + if err != nil { + logger.Error("reading record from event blocks (CAR)", "err", err) + break + } + if op.Cid == nil || lexutil.LexLink(rc) != *op.Cid { + logger.Error("mismatch between commit op CID and record block", "recordCID", rc, "opCID", op.Cid) + break + } + var action string + switch ek { + case repomgr.EvtKindCreateRecord: + action = automod.CreateOp + case repomgr.EvtKindUpdateRecord: + action = automod.UpdateOp + default: + logger.Error("impossible event kind", "kind", ek) + break + } + recCID := syntax.CID(op.Cid.String()) + op := automod.RecordOp{ + Action: action, + DID: did, + Collection: collection, + RecordKey: rkey, + CID: &recCID, + RecordCBOR: *recCBOR, + } + err = fc.Engine.ProcessRecordOp(context.Background(), op) + if err != nil { + logger.Error("engine failed to process record", "err", err) + continue + } + case repomgr.EvtKindDeleteRecord: + op := automod.RecordOp{ + Action: automod.DeleteOp, + DID: did, + Collection: collection, + RecordKey: rkey, + CID: nil, + RecordCBOR: nil, + } + err = fc.Engine.ProcessRecordOp(context.Background(), op) + if err != nil { + logger.Error("engine failed to process record", "err", err) + continue + } + default: + // TODO: should this be an error? + } + } + + return nil +} + +func (fc *FirehoseConsumer) ReadLastCursor(ctx context.Context) (int64, error) { + // if redis isn't configured, just skip + if fc.RedisClient == nil { + fc.Logger.Info("redis not configured, skipping cursor read") + return 0, nil + } + + val, err := fc.RedisClient.Get(ctx, firehoseCursorKey).Int64() + if err == redis.Nil { + fc.Logger.Info("no pre-existing cursor in redis") + return 0, nil + } else if err != nil { + return 0, err + } + fc.Logger.Info("successfully found prior subscription cursor seq in redis", "seq", val) + return val, nil +} + +func (fc *FirehoseConsumer) PersistCursor(ctx context.Context) error { + // if redis isn't configured, just skip + if fc.RedisClient == nil { + return nil + } + lastSeq := atomic.LoadInt64(&fc.lastSeq) + if lastSeq <= 0 { + return nil + } + err := fc.RedisClient.Set(ctx, firehoseCursorKey, lastSeq, 14*24*time.Hour).Err() + return err +} + +// this method runs in a loop, persisting the current cursor state every 5 seconds +func (fc *FirehoseConsumer) RunPersistCursor(ctx context.Context) error { + + // if redis isn't configured, just skip + if fc.RedisClient == nil { + return nil + } + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + lastSeq := atomic.LoadInt64(&fc.lastSeq) + if lastSeq >= 1 { + fc.Logger.Info("persisting final cursor seq value", "seq", lastSeq) + err := fc.PersistCursor(ctx) + if err != nil { + fc.Logger.Error("failed to persist cursor", "err", err, "seq", lastSeq) + } + } + return nil + case <-ticker.C: + lastSeq := atomic.LoadInt64(&fc.lastSeq) + if lastSeq >= 1 { + err := fc.PersistCursor(ctx) + if err != nil { + fc.Logger.Error("failed to persist cursor", "err", err, "seq", lastSeq) + } + } + } + } +} diff --git a/automod/consumer/ozone.go b/automod/consumer/ozone.go new file mode 100644 index 000000000..ddb8ec118 --- /dev/null +++ b/automod/consumer/ozone.go @@ -0,0 +1,192 @@ +package consumer + +import ( + "context" + "fmt" + "log/slog" + "sync/atomic" + "time" + + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/redis/go-redis/v9" +) + +// TODO: should probably make this not hepa-specific; or even configurable +var ozoneCursorKey = "hepa/ozoneTimestamp" + +type OzoneConsumer struct { + Logger *slog.Logger + RedisClient *redis.Client + OzoneClient *xrpc.Client + Engine *automod.Engine + + // same as lastSeq, but for Ozone timestamp cursor. the value is a string. + lastCursor atomic.Value +} + +func (oc *OzoneConsumer) Run(ctx context.Context) error { + + if oc.Engine == nil { + return fmt.Errorf("nil engine") + } + if oc.OzoneClient == nil { + return fmt.Errorf("nil ozoneclient") + } + + cur, err := oc.ReadLastCursor(ctx) + if err != nil { + return err + } + + if cur == "" { + cur = syntax.DatetimeNow().String() + } + since, err := syntax.ParseDatetime(cur) + if err != nil { + return err + } + + oc.Logger.Info("subscribing to ozone event log", "upstream", oc.OzoneClient.Host, "cursor", cur, "since", since) + var limit int64 = 50 + period := time.Second * 5 + + for { + me, err := toolsozone.ModerationQueryEvents( + ctx, + oc.OzoneClient, + nil, // addedLabels []string + nil, // addedTags []string + "", // ageAssuranceState + "", // batchId string + nil, // collections []string + "", // comment string + since.String(), // createdAfter string + "", // createdBefore string + "", // createdBy string + "", // cursor string + false, // hasComment bool + true, // includeAllUserRecords bool + limit, // limit int64 + nil, // modTool + nil, // policies []string + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "asc", // sortDirection string + "", // subject string + "", // subjectType string + nil, // types []string + false, // withStrike bool + ) + if err != nil { + oc.Logger.Warn("ozone query events failed; sleeping then will retrying", "err", err, "period", period.String()) + time.Sleep(period) + continue + } + + // track if the response contained anything new + anyNewEvents := false + for _, evt := range me.Events { + createdAt, err := syntax.ParseDatetime(evt.CreatedAt) + if err != nil { + return fmt.Errorf("invalid time format for ozone 'createdAt': %w", err) + } + // skip if the timestamp is the exact same + if createdAt == since { + continue + } + anyNewEvents = true + // TODO: is there a race condition here? + if !createdAt.Time().After(since.Time()) { + oc.Logger.Error("out of order ozone event", "createdAt", createdAt, "since", since) + return fmt.Errorf("out of order ozone event") + } + if err = oc.HandleOzoneEvent(ctx, evt); err != nil { + oc.Logger.Error("failed to process ozone event", "event", evt) + } + since = createdAt + oc.lastCursor.Store(since.String()) + } + if !anyNewEvents { + oc.Logger.Debug("... ozone poller sleeping", "period", period.String()) + time.Sleep(period) + } + } +} + +func (oc *OzoneConsumer) HandleOzoneEvent(ctx context.Context, eventView *toolsozone.ModerationDefs_ModEventView) error { + + oc.Logger.Debug("received ozone event", "eventID", eventView.Id, "createdAt", eventView.CreatedAt) + + if err := oc.Engine.ProcessOzoneEvent(context.Background(), eventView); err != nil { + oc.Logger.Error("engine failed to process ozone event", "err", err) + } + return nil +} + +func (oc *OzoneConsumer) ReadLastCursor(ctx context.Context) (string, error) { + // if redis isn't configured, just skip + if oc.RedisClient == nil { + oc.Logger.Info("redis not configured, skipping ozone cursor read") + return "", nil + } + + val, err := oc.RedisClient.Get(ctx, ozoneCursorKey).Result() + if err == redis.Nil || val == "" { + oc.Logger.Info("no pre-existing ozone cursor in redis") + return "", nil + } else if err != nil { + return "", err + } + oc.Logger.Info("successfully found prior ozone offset timestamp in redis", "cursor", val) + return val, nil +} + +func (oc *OzoneConsumer) PersistCursor(ctx context.Context) error { + // if redis isn't configured, just skip + if oc.RedisClient == nil { + return nil + } + lastCursor := oc.lastCursor.Load() + if lastCursor == nil || lastCursor == "" { + return nil + } + err := oc.RedisClient.Set(ctx, ozoneCursorKey, lastCursor, 14*24*time.Hour).Err() + return err +} + +// this method runs in a loop, persisting the current cursor state every 5 seconds +func (oc *OzoneConsumer) RunPersistCursor(ctx context.Context) error { + + // if redis isn't configured, just skip + if oc.RedisClient == nil { + return nil + } + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + lastCursor := oc.lastCursor.Load() + if lastCursor != nil && lastCursor != "" { + oc.Logger.Info("persisting final ozone cursor timestamp", "cursor", lastCursor) + err := oc.PersistCursor(ctx) + if err != nil { + oc.Logger.Error("failed to persist ozone cursor", "err", err, "cursor", lastCursor) + } + } + return nil + case <-ticker.C: + lastCursor := oc.lastCursor.Load() + if lastCursor != nil && lastCursor != "" { + err := oc.PersistCursor(ctx) + if err != nil { + oc.Logger.Error("failed to persist ozone cursor", "err", err, "cursor", lastCursor) + } + } + } + } +} diff --git a/automod/countstore/countstore.go b/automod/countstore/countstore.go new file mode 100644 index 000000000..a3ae2b9d1 --- /dev/null +++ b/automod/countstore/countstore.go @@ -0,0 +1,67 @@ +package countstore + +import ( + "context" + "fmt" + "log/slog" + "time" +) + +const ( + PeriodTotal = "total" + PeriodDay = "day" + PeriodHour = "hour" +) + +// CountStore is an interface for storing incrementing event counts, bucketed into periods. +// It is implemented by MemCountStore and by RedisCountStore. +// +// Period bucketing works on the basis of the current date (as determined mid-call). +// See the `Period*` consts for the available period types. +// +// The "GetCount" and "Increment" methods perform actual counting. +// The "*Distinct" methods have a different behavior: +// "IncrementDistinct" marks a value as seen at least once, +// and "GetCountDistinct" asks how _many_ values have been seen at least once. +// +// Incrementing -- both the "Increment" and "IncrementDistinct" variants -- increases +// a count in each supported period bucket size. +// In other words, one call to CountStore.Increment causes three increments internally: +// one to the count for the hour, one to the count for the day, and one to the all-time count. +// The "IncrementPeriod" method allows only incrementing a single period bucket. Care must be taken to match the "GetCount" period with the incremented period when using this variant. +// +// The exact implementation and precision of the "*Distinct" methods may vary: +// in the MemCountStore implementation, it is precise (it's based on large maps); +// in the RedisCountStore implementation, it uses the Redis "pfcount" feature, +// which is based on a HyperLogLog datastructure which has probabilistic properties +// (see https://redis.io/commands/pfcount/ ). +// +// Memory growth and availability of information over time also varies by implementation. +// The RedisCountStore implementation uses Redis's key expiration primitives; +// only the all-time counts go without expiration. +// The MemCountStore grows without bound (it's intended to be used in testing +// and other non-production operations). +type CountStore interface { + GetCount(ctx context.Context, name, val, period string) (int, error) + Increment(ctx context.Context, name, val string) error + IncrementPeriod(ctx context.Context, name, val, period string) error + // TODO: batch increment method + GetCountDistinct(ctx context.Context, name, bucket, period string) (int, error) + IncrementDistinct(ctx context.Context, name, bucket, val string) error +} + +func periodBucket(name, val, period string) string { + switch period { + case PeriodTotal: + return fmt.Sprintf("%s/%s", name, val) + case PeriodDay: + t := time.Now().UTC().Format(time.DateOnly) + return fmt.Sprintf("%s/%s/%s", name, val, t) + case PeriodHour: + t := time.Now().UTC().Format(time.RFC3339)[0:13] + return fmt.Sprintf("%s/%s/%s", name, val, t) + default: + slog.Warn("unhandled counter period", "period", period) + return fmt.Sprintf("%s/%s", name, val) + } +} diff --git a/automod/countstore/countstore_mem.go b/automod/countstore/countstore_mem.go new file mode 100644 index 000000000..f33c076ee --- /dev/null +++ b/automod/countstore/countstore_mem.go @@ -0,0 +1,71 @@ +package countstore + +import ( + "context" + + "github.com/puzpuzpuz/xsync/v3" +) + +type MemCountStore struct { + // Counts is keyed by a string that is a munge of "{name}/{val}[/{period}]", + // where period is either absent (meaning all-time total) + // or a string describing that timeperiod (either "YYYY-MM-DD" or that plus a literal "T" and "HH"). + // + // (Using a values for `name` and `val` with slashes in them is perhaps inadvisable, as it may be ambiguous.) + Counts *xsync.MapOf[string, int] + DistinctCounts *xsync.MapOf[string, *xsync.MapOf[string, bool]] +} + +func NewMemCountStore() MemCountStore { + return MemCountStore{ + Counts: xsync.NewMapOf[string, int](), + DistinctCounts: xsync.NewMapOf[string, *xsync.MapOf[string, bool]](), + } +} + +func (s MemCountStore) GetCount(ctx context.Context, name, val, period string) (int, error) { + v, ok := s.Counts.Load(periodBucket(name, val, period)) + if !ok { + return 0, nil + } + return v, nil +} + +func (s MemCountStore) Increment(ctx context.Context, name, val string) error { + for _, p := range []string{PeriodTotal, PeriodDay, PeriodHour} { + if err := s.IncrementPeriod(ctx, name, val, p); err != nil { + return err + } + } + return nil +} + +func (s MemCountStore) IncrementPeriod(ctx context.Context, name, val, period string) error { + k := periodBucket(name, val, period) + s.Counts.Compute(k, func(oldVal int, _ bool) (int, bool) { + return oldVal + 1, false + }) + return nil +} + +func (s MemCountStore) GetCountDistinct(ctx context.Context, name, bucket, period string) (int, error) { + v, ok := s.DistinctCounts.Load(periodBucket(name, bucket, period)) + if !ok { + return 0, nil + } + return v.Size(), nil +} + +func (s MemCountStore) IncrementDistinct(ctx context.Context, name, bucket, val string) error { + for _, p := range []string{PeriodTotal, PeriodDay, PeriodHour} { + k := periodBucket(name, bucket, p) + s.DistinctCounts.Compute(k, func(nested *xsync.MapOf[string, bool], _ bool) (*xsync.MapOf[string, bool], bool) { + if nested == nil { + nested = xsync.NewMapOf[string, bool]() + } + nested.Store(val, true) + return nested, false + }) + } + return nil +} diff --git a/automod/countstore/countstore_redis.go b/automod/countstore/countstore_redis.go new file mode 100644 index 000000000..2e42c96f8 --- /dev/null +++ b/automod/countstore/countstore_redis.go @@ -0,0 +1,121 @@ +package countstore + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" +) + +var redisCountPrefix string = "count/" +var redisDistinctPrefix string = "distinct/" + +type RedisCountStore struct { + Client *redis.Client +} + +func NewRedisCountStore(redisURL string) (*RedisCountStore, error) { + ctx := context.Background() + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + rdb := redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(ctx).Result() + if err != nil { + return nil, err + } + rcs := RedisCountStore{ + Client: rdb, + } + return &rcs, nil +} + +func (s *RedisCountStore) GetCount(ctx context.Context, name, val, period string) (int, error) { + key := redisCountPrefix + periodBucket(name, val, period) + c, err := s.Client.Get(ctx, key).Int() + if err == redis.Nil { + return 0, nil + } else if err != nil { + return 0, err + } + return c, nil +} + +func (s *RedisCountStore) Increment(ctx context.Context, name, val string) error { + + var key string + + // increment multiple counters in a single redis round-trip + multi := s.Client.Pipeline() + + key = redisCountPrefix + periodBucket(name, val, PeriodHour) + multi.Incr(ctx, key) + multi.Expire(ctx, key, 2*time.Hour) + + key = redisCountPrefix + periodBucket(name, val, PeriodDay) + multi.Incr(ctx, key) + multi.Expire(ctx, key, 48*time.Hour) + + key = redisCountPrefix + periodBucket(name, val, PeriodTotal) + multi.Incr(ctx, key) + // no expiration for total + + _, err := multi.Exec(ctx) + return err +} + +// Variant of Increment() which only acts on a single specified time period. The intended us of this variant is to control the total number of counters persisted, by using a relatively short time period, for which the counters will expire. +func (s *RedisCountStore) IncrementPeriod(ctx context.Context, name, val, period string) error { + + // multiple ops in a single redis round-trip + multi := s.Client.Pipeline() + + key := redisCountPrefix + periodBucket(name, val, period) + multi.Incr(ctx, key) + + switch period { + case PeriodHour: + multi.Expire(ctx, key, 2*time.Hour) + case PeriodDay: + multi.Expire(ctx, key, 48*time.Hour) + } + + _, err := multi.Exec(ctx) + return err +} + +func (s *RedisCountStore) GetCountDistinct(ctx context.Context, name, val, period string) (int, error) { + key := redisDistinctPrefix + periodBucket(name, val, period) + c, err := s.Client.PFCount(ctx, key).Result() + if err == redis.Nil { + return 0, nil + } else if err != nil { + return 0, err + } + return int(c), nil +} + +func (s *RedisCountStore) IncrementDistinct(ctx context.Context, name, bucket, val string) error { + + var key string + + // increment multiple counters in a single redis round-trip + multi := s.Client.Pipeline() + + key = redisDistinctPrefix + periodBucket(name, bucket, PeriodHour) + multi.PFAdd(ctx, key, val) + multi.Expire(ctx, key, 2*time.Hour) + + key = redisDistinctPrefix + periodBucket(name, bucket, PeriodDay) + multi.PFAdd(ctx, key, val) + multi.Expire(ctx, key, 48*time.Hour) + + key = redisDistinctPrefix + periodBucket(name, bucket, PeriodTotal) + multi.PFAdd(ctx, key, val) + // no expiration for total + + _, err := multi.Exec(ctx) + return err +} diff --git a/automod/countstore/countstore_test.go b/automod/countstore/countstore_test.go new file mode 100644 index 000000000..e9d1d5665 --- /dev/null +++ b/automod/countstore/countstore_test.go @@ -0,0 +1,106 @@ +package countstore + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestMemCountStoreBasics(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + cs := NewMemCountStore() + + c, err := cs.GetCount(ctx, "test1", "val1", PeriodTotal) + assert.NoError(err) + assert.Equal(0, c) + assert.NoError(cs.Increment(ctx, "test1", "val1")) + assert.NoError(cs.Increment(ctx, "test1", "val1")) + + for _, period := range []string{PeriodTotal, PeriodDay, PeriodHour} { + c, err = cs.GetCount(ctx, "test1", "val1", period) + assert.NoError(err) + assert.Equal(2, c) + } + + c, err = cs.GetCountDistinct(ctx, "test2", "val2", PeriodTotal) + assert.NoError(err) + assert.Equal(0, c) + assert.NoError(cs.IncrementDistinct(ctx, "test2", "val2", "one")) + assert.NoError(cs.IncrementDistinct(ctx, "test2", "val2", "one")) + assert.NoError(cs.IncrementDistinct(ctx, "test2", "val2", "one")) + c, err = cs.GetCountDistinct(ctx, "test2", "val2", PeriodTotal) + assert.NoError(err) + assert.Equal(1, c) + + assert.NoError(cs.IncrementDistinct(ctx, "test2", "val2", "two")) + assert.NoError(cs.IncrementDistinct(ctx, "test2", "val2", "three")) + + for _, period := range []string{PeriodTotal, PeriodDay, PeriodHour} { + c, err = cs.GetCountDistinct(ctx, "test2", "val2", period) + assert.NoError(err) + assert.Equal(3, c) + } +} + +func TestMemCountStoreConcurrent(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + cs := NewMemCountStore() + + c, err := cs.GetCount(ctx, "test1", "val1", PeriodTotal) + assert.NoError(err) + assert.Equal(0, c) + + // Increment two different values from four different goroutines. + // Read from two more (don't assert values; just that there's no error, + // and no race (run this with `-race`!). + // A short sleep ensures the scheduler is yielded to, so that order is decently random, + // and reads are interleaved with writes. + var wg sync.WaitGroup + fnInc := func(name, val string, times int) { + for i := 0; i < times; i++ { + assert.NoError(cs.Increment(ctx, name, val)) + assert.NoError(cs.IncrementDistinct(ctx, name, name, val)) + time.Sleep(time.Nanosecond) + } + wg.Done() + } + fnRead := func(name, val string, times int) { + for i := 0; i < times; i++ { + _, err := cs.GetCount(ctx, name, val, PeriodTotal) + assert.NoError(err) + time.Sleep(time.Nanosecond) + } + } + wg.Add(4) + go fnInc("test1", "val1", 10) + go fnInc("test1", "val1", 10) + go fnRead("test1", "val1", 10) + go fnInc("test2", "val2", 6) + go fnInc("test2", "val2", 6) + go fnRead("test2", "val2", 6) + wg.Wait() + + // One final read for each value after all writer routines are collected. + // This one should match a fixed value of the sum of all writes. + c, err = cs.GetCount(ctx, "test1", "val1", PeriodTotal) + assert.NoError(err) + assert.Equal(20, c) + c, err = cs.GetCount(ctx, "test2", "val2", PeriodTotal) + assert.NoError(err) + assert.Equal(12, c) + + // And what of distinct counts? Those should be 1. + c, err = cs.GetCountDistinct(ctx, "test1", "test1", PeriodTotal) + assert.NoError(err) + assert.Equal(1, c) + c, err = cs.GetCountDistinct(ctx, "test2", "test2", PeriodTotal) + assert.NoError(err) + assert.Equal(1, c) +} diff --git a/automod/countstore/doc.go b/automod/countstore/doc.go new file mode 100644 index 000000000..66becbfa6 --- /dev/null +++ b/automod/countstore/doc.go @@ -0,0 +1,2 @@ +// Interface for fast atomic counters, and separate implementations using redis and in-process memory. +package countstore diff --git a/automod/doc.go b/automod/doc.go new file mode 100644 index 000000000..e6c025f88 --- /dev/null +++ b/automod/doc.go @@ -0,0 +1,6 @@ +// Auto-Moderation rules engine for anti-spam and other moderation tasks. +// +// This package (`github.com/bluesky-social/indigo/automod`) contains a "rules engine" to augment human moderators in the atproto network. Batches of rules are processed for novel "events" such as a new post or update of an account handle. Counters and other statistics are collected, which can drive subsequent rule invocations. The outcome of rules can be moderation events like "report account for human review" or "label post". A lot of what this package does is collect and maintain caches of relevant metadata about accounts and pieces of content, so that rules have efficient access to this information. +// +// See `automod/README.md` for more background, and `cmd/hepa` for a daemon built on this package. +package automod diff --git a/automod/engine/account_meta.go b/automod/engine/account_meta.go new file mode 100644 index 000000000..9e3ca15c1 --- /dev/null +++ b/automod/engine/account_meta.go @@ -0,0 +1,56 @@ +package engine + +import ( + "time" + + "github.com/bluesky-social/indigo/atproto/identity" +) + +var ( + ReviewStateEscalated = "escalated" + ReviewStateOpen = "open" + ReviewStateClosed = "closed" + ReviewStateNone = "none" +) + +// information about a repo/account/identity, always pre-populated and relevant to many rules +type AccountMeta struct { + Identity *identity.Identity + Profile ProfileSummary + Private *AccountPrivate + AccountLabels []string + AccountNegatedLabels []string + AccountFlags []string + FollowersCount int64 + FollowsCount int64 + PostsCount int64 + Takendown bool + Deactivated bool + // best effort public interpretation of account creation timestamp. not always available, and may be inaccurate/inconsistent for now. + CreatedAt *time.Time +} + +type ProfileSummary struct { + HasAvatar bool + AvatarCid *string + BannerCid *string + Description *string + DisplayName *string +} + +// opaque fingerprints for correlating abusive accounts +type AbuseSignature struct { + Property string + Value string +} + +type AccountPrivate struct { + Email string + EmailConfirmed bool + IndexedAt *time.Time + AccountTags []string + // ReviewState will be one of ReviewStateEscalated, ReviewStateOpen, ReviewStateClosed, ReviewStateNone, or "" (unknown) + ReviewState string + Appealed bool + AbuseSignatures []AbuseSignature +} diff --git a/automod/engine/action_dedupe_test.go b/automod/engine/action_dedupe_test.go new file mode 100644 index 000000000..b07a2a430 --- /dev/null +++ b/automod/engine/action_dedupe_test.go @@ -0,0 +1,58 @@ +package engine + +import ( + "bytes" + "context" + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod/countstore" + + "github.com/stretchr/testify/assert" +) + +func alwaysReportAccountRule(c *RecordContext) error { + c.ReportAccount(ReportReasonOther, "test report") + return nil +} + +func TestAccountReportDedupe(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + eng := EngineTestFixture() + eng.Rules = RuleSet{ + RecordRules: []RecordRuleFunc{ + alwaysReportAccountRule, + }, + } + + //path := "app.bsky.feed.post/abc123" + cid1 := syntax.CID("cid123") + p1 := appbsky.FeedPost{Text: "some post blah"} + p1buf := new(bytes.Buffer) + assert.NoError(p1.MarshalCBOR(p1buf)) + p1cbor := p1buf.Bytes() + id1 := identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + } + + // exact same event multiple times; should only report once + op := RecordOp{ + Action: CreateOp, + DID: id1.DID, + Collection: "app.bsky.feed.post", + RecordKey: "abc123", + CID: &cid1, + RecordCBOR: p1cbor, + } + for i := 0; i < 5; i++ { + assert.NoError(eng.ProcessRecordOp(ctx, op)) + } + + reports, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay) + assert.NoError(err) + assert.Equal(1, reports) +} diff --git a/automod/engine/blobs.go b/automod/engine/blobs.go new file mode 100644 index 000000000..3efb90c28 --- /dev/null +++ b/automod/engine/blobs.go @@ -0,0 +1,95 @@ +package engine + +import ( + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/bluesky-social/indigo/atproto/atdata" + lexutil "github.com/bluesky-social/indigo/lex/util" + + "github.com/earthboundkid/versioninfo/v2" +) + +// Parses out any blobs from the enclosed record. +// +// NOTE: for consistency with other RecordContext methods, which don't usually return errors, maybe the error-returning version of this function should be a helper function, or defined on RecordOp, and the RecordContext version should return an empty array on error? +func (c *RecordContext) Blobs() ([]lexutil.LexBlob, error) { + + if c.RecordOp.Action == DeleteOp { + return []lexutil.LexBlob{}, nil + } + + rec, err := atdata.UnmarshalCBOR(c.RecordOp.RecordCBOR) + if err != nil { + return nil, fmt.Errorf("parsing generic record CBOR: %v", err) + } + blobs := atdata.ExtractBlobs(rec) + + // convert from atdata.Blob to lexutil.LexBlob; plan is to merge these types eventually + var out []lexutil.LexBlob + for _, b := range blobs { + lb := lexutil.LexBlob{ + Ref: lexutil.LexLink(b.Ref), + MimeType: b.MimeType, + Size: b.Size, + } + out = append(out, lb) + } + return out, nil +} + +func (c *RecordContext) fetchBlob(blob lexutil.LexBlob) ([]byte, error) { + + start := time.Now() + defer func() { + duration := time.Since(start) + blobDownloadDuration.Observe(duration.Seconds()) + }() + + var blobBytes []byte + + // TODO: potential security issue here with malformed or "localhost" PDS endpoint + pdsEndpoint := c.Account.Identity.PDSEndpoint() + xrpcURL := fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob?did=%s&cid=%s", pdsEndpoint, c.Account.Identity.DID, blob.Ref) + + req, err := http.NewRequest("GET", xrpcURL, nil) + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", "indigo-automod/"+versioninfo.Short()) + // TODO: more robust PDS hostname check (eg, future trailing slash or partial path) + if c.engine.BskyClient.Headers != nil && strings.HasSuffix(pdsEndpoint, ".bsky.network") { + val, ok := c.engine.BskyClient.Headers["x-ratelimit-bypass"] + if ok { + req.Header.Set("x-ratelimit-bypass", val) + } + } + + client := c.engine.BlobClient + if client == nil { + client = http.DefaultClient + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + blobDownloadCount.WithLabelValues(fmt.Sprint(resp.StatusCode)).Inc() + if resp.StatusCode != 200 { + io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("failed to fetch blob from PDS. did=%s cid=%s statusCode=%d", c.Account.Identity.DID, blob.Ref, resp.StatusCode) + } + + blobBytes, err = io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return blobBytes, nil +} diff --git a/automod/engine/cid_from_cdn_test.go b/automod/engine/cid_from_cdn_test.go new file mode 100644 index 000000000..0780403ca --- /dev/null +++ b/automod/engine/cid_from_cdn_test.go @@ -0,0 +1,42 @@ +package engine + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCidFromCdnUrl(t *testing.T) { + assert := assert.New(t) + + fixCid := "abcdefghijk" + + fixtures := []struct { + url string + cid *string + }{ + { + url: "https://cdn.bsky.app/img/avatar/plain/did:plc:abc123/abcdefghijk@jpeg", + cid: &fixCid, + }, + { + url: "https://cdn.bsky.app/img/feed_fullsize/plain/did:plc:abc123/abcdefghijk@jpeg", + cid: &fixCid, + }, + { + url: "https://cdn.bsky.app/img/feed_fullsize", + cid: nil, + }, + { + url: "https://cdn.bsky.app/img/feed_fullsize/plain/did:plc:abc123/abcdefghijk", + cid: &fixCid, + }, + { + url: "https://cdn.asky.app/img/feed_fullsize/plain/did:plc:abc123/abcdefghijk@jpeg", + cid: nil, + }, + } + + for _, fix := range fixtures { + assert.Equal(fix.cid, cidFromCdnUrl(&fix.url)) + } +} diff --git a/automod/engine/circuit_breaker_test.go b/automod/engine/circuit_breaker_test.go new file mode 100644 index 000000000..dee3dcc52 --- /dev/null +++ b/automod/engine/circuit_breaker_test.go @@ -0,0 +1,116 @@ +package engine + +import ( + "bytes" + "context" + "fmt" + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod/countstore" + + "github.com/stretchr/testify/assert" +) + +func alwaysTakedownRecordRule(c *RecordContext) error { + c.TakedownRecord() + return nil +} + +func alwaysReportRecordRule(c *RecordContext) error { + c.ReportRecord(ReportReasonOther, "test report") + return nil +} + +func TestTakedownCircuitBreaker(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + eng := EngineTestFixture() + dir := identity.NewMockDirectory() + eng.Directory = &dir + // note that this is a record-level action, not account-level + eng.Rules = RuleSet{ + RecordRules: []RecordRuleFunc{ + alwaysTakedownRecordRule, + }, + } + + cid1 := syntax.CID("cid123") + p1 := appbsky.FeedPost{Text: "some post blah"} + p1buf := new(bytes.Buffer) + assert.NoError(p1.MarshalCBOR(p1buf)) + p1cbor := p1buf.Bytes() + + // generate double the quote of events; expect to only count the quote worth of actions + for i := 0; i < 2*eng.Config.QuotaModTakedownDay; i++ { + ident := identity.Identity{ + DID: syntax.DID(fmt.Sprintf("did:plc:abc%d", i)), + Handle: syntax.Handle("handle.example.com"), + } + dir.Insert(ident) + op := RecordOp{ + Action: CreateOp, + DID: ident.DID, + Collection: syntax.NSID("app.bsky.feed.post"), + RecordKey: syntax.RecordKey("abc123"), + CID: &cid1, + RecordCBOR: p1cbor, + } + assert.NoError(eng.ProcessRecordOp(ctx, op)) + } + + takedowns, err := eng.Counters.GetCount(ctx, "automod-quota", "takedown", countstore.PeriodDay) + assert.NoError(err) + assert.Equal(eng.Config.QuotaModTakedownDay, takedowns) + + reports, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay) + assert.NoError(err) + assert.Equal(0, reports) +} + +func TestReportCircuitBreaker(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + eng := EngineTestFixture() + dir := identity.NewMockDirectory() + eng.Directory = &dir + eng.Rules = RuleSet{ + RecordRules: []RecordRuleFunc{ + alwaysReportRecordRule, + }, + } + + cid1 := syntax.CID("cid123") + p1 := appbsky.FeedPost{Text: "some post blah"} + p1buf := new(bytes.Buffer) + assert.NoError(p1.MarshalCBOR(p1buf)) + p1cbor := p1buf.Bytes() + + // generate double the quota of events; expect to only count the quota worth of actions + for i := 0; i < 2*eng.Config.QuotaModReportDay; i++ { + ident := identity.Identity{ + DID: syntax.DID(fmt.Sprintf("did:plc:abc%d", i)), + Handle: syntax.Handle("handle.example.com"), + } + dir.Insert(ident) + op := RecordOp{ + Action: CreateOp, + DID: ident.DID, + Collection: syntax.NSID("app.bsky.feed.post"), + RecordKey: syntax.RecordKey("abc123"), + CID: &cid1, + RecordCBOR: p1cbor, + } + assert.NoError(eng.ProcessRecordOp(ctx, op)) + } + + takedowns, err := eng.Counters.GetCount(ctx, "automod-quota", "takedown", countstore.PeriodDay) + assert.NoError(err) + assert.Equal(0, takedowns) + + reports, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay) + assert.NoError(err) + assert.Equal(eng.Config.QuotaModReportDay, reports) +} diff --git a/automod/engine/context.go b/automod/engine/context.go new file mode 100644 index 000000000..2fd17b78f --- /dev/null +++ b/automod/engine/context.go @@ -0,0 +1,303 @@ +package engine + +import ( + "context" + "fmt" + "log/slog" + + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// The primary interface exposed to rules. All other contexts derive from this "base" struct. +type BaseContext struct { + // Actual golang "context.Context", if needed for timeouts etc + Ctx context.Context + // Any errors encountered while processing methods on this struct (or sub-types) get rolled up in this nullable field + Err error + // slog logger handle, with event-specific structured fields pre-populated. Pointer, but expected to never be nil. + Logger *slog.Logger + + engine *Engine // NOTE: pointer, but expected never to be nil + effects *Effects +} + +// Both a useful context on it's own (eg, for identity events), and extended by other context types. +type AccountContext struct { + BaseContext + + Account AccountMeta +} + +// Represents a repository operation on a single record: create, update, delete, etc. +type RecordContext struct { + AccountContext + + RecordOp RecordOp + // TODO: could consider adding commit-level metadata here. probably nullable if so, commit-level metadata isn't always available. might be best to do a separate event/context type for that +} + +// Represents an ozone event on a subject account. +// +// TODO: for ozone events with a record subject (not account subject), should we extend RecordContext instead? +type OzoneEventContext struct { + AccountContext + + Event OzoneEvent + + // Moderator team member (for ozone internal events) or account that created a report or appeal + CreatorAccount AccountMeta + + // If the subject of the event is a record, this is the record metadata + SubjectRecord *RecordMeta +} + +var ( + CreateOp = "create" + UpdateOp = "update" + DeleteOp = "delete" +) + +// Immutable +type RecordOp struct { + // Indicates type of record mutation: create, update, or delete. + // The term "action" is copied from com.atproto.sync.subscribeRepos#repoOp + Action string + DID syntax.DID + Collection syntax.NSID + RecordKey syntax.RecordKey + CID *syntax.CID + RecordCBOR []byte +} + +// Immutable +type RecordMeta struct { + DID syntax.DID + Collection syntax.NSID + RecordKey syntax.RecordKey + CID *syntax.CID + // TODO: RecordCBOR []byte? optional? +} + +type OzoneEvent struct { + EventType string + EventID int64 + CreatedAt syntax.Datetime + CreatedBy syntax.DID + SubjectDID syntax.DID + SubjectURI *syntax.ATURI + // TODO: SubjectBlobs []syntax.CID + Event toolsozone.ModerationDefs_ModEventView_Event +} + +// Checks that op has expected fields, based on the action type +func (op *RecordOp) Validate() error { + switch op.Action { + case CreateOp, UpdateOp: + if op.RecordCBOR == nil || op.CID == nil { + return fmt.Errorf("expected record create/update op to contain both value and CID") + } + case DeleteOp: + if op.RecordCBOR != nil || op.CID != nil { + return fmt.Errorf("expected record delete op to be empty") + } + default: + return fmt.Errorf("unexpected record op action: %s", op.Action) + } + return nil +} + +func (op *RecordOp) ATURI() syntax.ATURI { + return syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", op.DID, op.Collection, op.RecordKey)) +} + +// TODO: in the future *may* have an IdentityContext with an IdentityOp sub-field + +// Access to engine's identity directory (without access to other engine fields) +func (c *BaseContext) Directory() identity.Directory { + return c.engine.Directory +} + +// request external state via engine (indirect) +func (c *BaseContext) GetCount(name, val, period string) int { + out, err := c.engine.Counters.GetCount(c.Ctx, name, val, period) + if err != nil { + if nil == c.Err { + c.Err = err + } + return 0 + } + return out +} + +func (c *BaseContext) GetCountDistinct(name, bucket, period string) int { + out, err := c.engine.Counters.GetCountDistinct(c.Ctx, name, bucket, period) + if err != nil { + if nil == c.Err { + c.Err = err + } + return 0 + } + return out +} + +func (c *BaseContext) InSet(name, val string) bool { + out, err := c.engine.Sets.InSet(c.Ctx, name, val) + if err != nil { + if nil == c.Err { + c.Err = err + } + return false + } + return out +} + +// Returns a pointer to the underlying automod engine. This usually should NOT be used in rules. +// +// This is an escape hatch for hacking on the system before features get fully integerated in to the content API surface. The Engine API is not stable. +func (c *BaseContext) InternalEngine() *Engine { + return c.engine +} + +func NewAccountContext(ctx context.Context, eng *Engine, meta AccountMeta) AccountContext { + return AccountContext{ + BaseContext: BaseContext{ + Ctx: ctx, + Err: nil, + Logger: eng.Logger.With("did", meta.Identity.DID), + engine: eng, + effects: &Effects{}, + }, + Account: meta, + } +} + +func NewRecordContext(ctx context.Context, eng *Engine, meta AccountMeta, op RecordOp) RecordContext { + ac := NewAccountContext(ctx, eng, meta) + ac.BaseContext.Logger = ac.BaseContext.Logger.With("collection", op.Collection, "rkey", op.RecordKey) + return RecordContext{ + AccountContext: ac, + RecordOp: op, + } +} + +// fetch relationship metadata between this account and another account +func (c *AccountContext) GetAccountRelationship(other syntax.DID) AccountRelationship { + rel, err := c.engine.GetAccountRelationship(c.Ctx, c.Account.Identity.DID, other) + if err != nil { + if nil == c.Err { + c.Err = err + } + return AccountRelationship{DID: other} + } + return *rel +} + +// fetch account metadata for the given DID. if there is any problem with lookup, returns nil. +// +// TODO: should this take an AtIdentifier instead? +func (c *BaseContext) GetAccountMeta(did syntax.DID) *AccountMeta { + + ident, err := c.engine.Directory.LookupDID(c.Ctx, did) + if err != nil { + if nil == c.Err { + c.Err = err + } + return nil + } + am, err := c.engine.GetAccountMeta(c.Ctx, ident) + if err != nil { + if nil == c.Err { + c.Err = err + } + return nil + } + return am +} + +// update effects (indirect) ====== + +func (c *BaseContext) Increment(name, val string) { + c.effects.Increment(name, val) +} + +func (c *BaseContext) IncrementDistinct(name, bucket, val string) { + c.effects.IncrementDistinct(name, bucket, val) +} + +func (c *BaseContext) IncrementPeriod(name, val string, period string) { + c.effects.IncrementPeriod(name, val, period) +} + +func (c *BaseContext) Notify(srv string) { + c.effects.Notify(srv) +} + +func (c *AccountContext) AddAccountFlag(val string) { + c.effects.AddAccountFlag(val) +} + +func (c *AccountContext) AddAccountLabel(val string) { + c.effects.AddAccountLabel(val) +} + +func (c *AccountContext) RemoveAccountLabel(val string) { + c.effects.RemoveAccountLabel(val) +} + +func (c *AccountContext) AddAccountTag(val string) { + c.effects.AddAccountTag(val) +} + +func (c *AccountContext) ReportAccount(reason, comment string) { + c.effects.ReportAccount(reason, comment) +} + +func (c *AccountContext) TakedownAccount() { + c.effects.TakedownAccount() +} + +func (c *AccountContext) EscalateAccount() { + c.effects.EscalateAccount() +} + +func (c *AccountContext) AcknowledgeAccount() { + c.effects.AcknowledgeAccount() +} + +func (c *RecordContext) AddRecordFlag(val string) { + c.effects.AddRecordFlag(val) +} + +func (c *RecordContext) AddRecordLabel(val string) { + c.effects.AddRecordLabel(val) +} + +func (c *RecordContext) RemoveRecordLabel(val string) { + c.effects.RemoveRecordLabel(val) +} + +func (c *RecordContext) AddRecordTag(val string) { + c.effects.AddRecordTag(val) +} + +func (c *RecordContext) ReportRecord(reason, comment string) { + c.effects.ReportRecord(reason, comment) +} + +func (c *RecordContext) TakedownRecord() { + c.effects.TakedownRecord() +} + +func (c *RecordContext) EscalateRecord() { + c.effects.EscalateRecord() +} + +func (c *RecordContext) AcknowledgeRecord() { + c.effects.AcknowledgeRecord() +} + +func (c *RecordContext) TakedownBlob(cid string) { + c.effects.TakedownBlob(cid) +} diff --git a/automod/engine/doc.go b/automod/engine/doc.go new file mode 100644 index 000000000..ef01f3801 --- /dev/null +++ b/automod/engine/doc.go @@ -0,0 +1,2 @@ +// Core automod rules engine implementation and related types +package engine diff --git a/automod/engine/effects.go b/automod/engine/effects.go new file mode 100644 index 000000000..44c6ae221 --- /dev/null +++ b/automod/engine/effects.go @@ -0,0 +1,275 @@ +package engine + +import ( + "sync" +) + +type CounterRef struct { + Name string + Val string + Period *string +} + +type CounterDistinctRef struct { + Name string + Bucket string + Val string +} + +// Mutable container for all the possible side-effects from rule execution. +// +// This single type tracks generic effects (eg, counter increments), account-level actions, and record-level actions (even for processing of account-level events which have no possible record-level effects). +type Effects struct { + // internal field for ensuring concurrent mutations are safe + mu sync.Mutex + // List of counters which should be incremented as part of processing this event. These are collected during rule execution and persisted in bulk at the end. + CounterIncrements []CounterRef + // Similar to "CounterIncrements", but for "distinct" style counters + CounterDistinctIncrements []CounterDistinctRef // TODO: better variable names + // Label values which should be applied to the overall account, as a result of rule execution. + AccountLabels []string + // Label values which should be removed from the overall account, as a result of rule execution. + RemovedAccountLabels []string + // Moderation tags (similar to labels, but private) which should be applied to the overall account, as a result of rule execution. + AccountTags []string + // automod flags (metadata) which should be applied to the account as a result of rule execution. + AccountFlags []string + // Reports which should be filed against this account, as a result of rule execution. + AccountReports []ModReport + // If "true", a rule decided that the entire account should have a takedown. + AccountTakedown bool + // If "true", a rule decided that the reported account should be escalated. + AccountEscalate bool + // If "true", a rule decided that the reports on account should be resolved as acknowledged. + AccountAcknowledge bool + // Same as "AccountLabels", but at record-level + RecordLabels []string + // Same as "RemovedRecordLabels", but at record-level + RemovedRecordLabels []string + // Same as "AccountTags", but at record-level + RecordTags []string + // Same as "AccountFlags", but at record-level + RecordFlags []string + // Same as "AccountReports", but at record-level + RecordReports []ModReport + // Same as "AccountTakedown", but at record-level + RecordTakedown bool + // Same as "AccountEscalate", but at record-level + RecordEscalate bool + // Same as "AccountAcknowledge", but at record-level + RecordAcknowledge bool + // Set of Blob CIDs to takedown (eg, purge from CDN) when doing a record takedown + BlobTakedowns []string + // If "true", indicates that a rule indicates that the action causing the event should be blocked or prevented + RejectEvent bool + // Services, if any, which should blast out a notification about this even (eg, Slack) + NotifyServices []string +} + +// Enqueues the named counter to be incremented at the end of all rule processing. Will automatically increment for all time periods. +// +// "name" is the counter namespace. +// "val" is the specific counter with that namespace. +func (e *Effects) Increment(name, val string) { + e.mu.Lock() + defer e.mu.Unlock() + e.CounterIncrements = append(e.CounterIncrements, CounterRef{Name: name, Val: val}) +} + +// Enqueues the named counter to be incremented at the end of all rule processing. Will only increment the indicated time period bucket. +func (e *Effects) IncrementPeriod(name, val string, period string) { + e.mu.Lock() + defer e.mu.Unlock() + e.CounterIncrements = append(e.CounterIncrements, CounterRef{Name: name, Val: val, Period: &period}) +} + +// Enqueues the named "distinct value" counter based on the supplied string value ("val") to be incremented at the end of all rule processing. Will automatically increment for all time periods. +func (e *Effects) IncrementDistinct(name, bucket, val string) { + e.mu.Lock() + defer e.mu.Unlock() + e.CounterDistinctIncrements = append(e.CounterDistinctIncrements, CounterDistinctRef{Name: name, Bucket: bucket, Val: val}) +} + +// Enqueues the provided label (string value) to be added to the account at the end of rule processing. +func (e *Effects) AddAccountLabel(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.AccountLabels { + if v == val { + return + } + } + e.AccountLabels = append(e.AccountLabels, val) +} + +// Enqueues the provided label (string value) to be removed from the account at the end of rule processing. +func (e *Effects) RemoveAccountLabel(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.RemovedAccountLabels { + if v == val { + return + } + } + e.RemovedAccountLabels = append(e.RemovedAccountLabels, val) +} + +// Enqueues the provided label (string value) to be added to the account at the end of rule processing. +func (e *Effects) AddAccountTag(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.AccountTags { + if v == val { + return + } + } + e.AccountTags = append(e.AccountTags, val) +} + +// Enqueues the provided flag (string value) to be recorded (in the Engine's flagstore) at the end of rule processing. +func (e *Effects) AddAccountFlag(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.AccountFlags { + if v == val { + return + } + } + e.AccountFlags = append(e.AccountFlags, val) +} + +// Enqueues a moderation report to be filed against the account at the end of rule processing. +func (e *Effects) ReportAccount(reason, comment string) { + e.mu.Lock() + defer e.mu.Unlock() + if comment == "" { + comment = "(reporting without comment)" + } + for _, v := range e.AccountReports { + if v.ReasonType == reason { + return + } + } + e.AccountReports = append(e.AccountReports, ModReport{ReasonType: reason, Comment: comment}) +} + +// Enqueues the entire account to be taken down at the end of rule processing. +func (e *Effects) TakedownAccount() { + e.AccountTakedown = true +} + +// Enqueues the account to be "escalated" for mod review at the end of rule processing. +func (e *Effects) EscalateAccount() { + e.AccountEscalate = true +} + +// Enqueues reports on account to be "acknowledged" (closed) at the end of rule processing. +func (e *Effects) AcknowledgeAccount() { + e.AccountAcknowledge = true +} + +// Enqueues the provided label (string value) to be added to the record at the end of rule processing. +func (e *Effects) AddRecordLabel(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.RecordLabels { + if v == val { + return + } + } + e.RecordLabels = append(e.RecordLabels, val) +} + +// Enqueues the provided label (string value) to be removed from the record at the end of rule processing. +func (e *Effects) RemoveRecordLabel(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.RemovedRecordLabels { + if v == val { + return + } + } + e.RemovedRecordLabels = append(e.RemovedRecordLabels, val) +} + +// Enqueues the provided tag (string value) to be added to the record at the end of rule processing. +func (e *Effects) AddRecordTag(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.RecordTags { + if v == val { + return + } + } + e.RecordTags = append(e.RecordTags, val) +} + +// Enqueues the provided flag (string value) to be recorded (in the Engine's flagstore) at the end of rule processing. +func (e *Effects) AddRecordFlag(val string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.RecordFlags { + if v == val { + return + } + } + e.RecordFlags = append(e.RecordFlags, val) +} + +// Enqueues a moderation report to be filed against the record at the end of rule processing. +func (e *Effects) ReportRecord(reason, comment string) { + e.mu.Lock() + defer e.mu.Unlock() + if comment == "" { + comment = "(reporting without comment)" + } + for _, v := range e.RecordReports { + if v.ReasonType == reason { + return + } + } + e.RecordReports = append(e.RecordReports, ModReport{ReasonType: reason, Comment: comment}) +} + +// Enqueues the record to be taken down at the end of rule processing. +func (e *Effects) TakedownRecord() { + e.RecordTakedown = true +} + +// Enqueues the record to be "escalated" for mod review at the end of rule processing. +func (e *Effects) EscalateRecord() { + e.RecordEscalate = true +} + +// Enqueues the record to be "escalated" for mod review at the end of rule processing. +func (e *Effects) AcknowledgeRecord() { + e.RecordAcknowledge = true +} + +// Enqueues the blob CID to be taken down (aka, CDN purge) as part of any record takedown +func (e *Effects) TakedownBlob(cid string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.BlobTakedowns { + if v == cid { + return + } + } + e.BlobTakedowns = append(e.BlobTakedowns, cid) +} + +// Records that the given service should be notified about this event +func (e *Effects) Notify(srv string) { + e.mu.Lock() + defer e.mu.Unlock() + for _, v := range e.NotifyServices { + if v == srv { + return + } + } + e.NotifyServices = append(e.NotifyServices, srv) +} + +func (e *Effects) Reject() { + e.RejectEvent = true +} diff --git a/automod/engine/engine.go b/automod/engine/engine.go new file mode 100644 index 000000000..c6d8bab6d --- /dev/null +++ b/automod/engine/engine.go @@ -0,0 +1,333 @@ +package engine + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod/cachestore" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/flagstore" + "github.com/bluesky-social/indigo/automod/setstore" + "github.com/bluesky-social/indigo/xrpc" +) + +// runtime for executing rules, managing state, and recording moderation actions. +// +// NOTE: careful when initializing: several fields must not be nil or zero, even though they are pointer type. +type Engine struct { + Logger *slog.Logger + Directory identity.Directory + Rules RuleSet + Counters countstore.CountStore + Sets setstore.SetStore + Cache cachestore.CacheStore + Flags flagstore.FlagStore + // unlike the other sub-modules, this field (Notifier) may be nil + Notifier Notifier + // use to fetch public account metadata from AppView; no auth + BskyClient *xrpc.Client + // used to persist moderation actions in ozone moderation service; optional, admin auth + OzoneClient *xrpc.Client + // used to fetch private account metadata from PDS or entryway; optional, admin auth + AdminClient *xrpc.Client + // used to fetch blobs from upstream PDS instances + BlobClient *http.Client + + // internal configuration + Config EngineConfig +} + +type EngineConfig struct { + // if enabled, account metadata is not hydrated for every event by default + SkipAccountMeta bool + // time period within which automod will not re-report an account for the same reasonType + ReportDupePeriod time.Duration + // number of reports automod can file per day, for all subjects and types combined (circuit breaker) + QuotaModReportDay int + // number of takedowns automod can action per day, for all subjects combined (circuit breaker) + QuotaModTakedownDay int + // number of misc actions automod can do per day, for all subjects combined (circuit breaker) + QuotaModActionDay int + + // timeout for record event processing (total, including all setup, rules, and teardown) + RecordEventTimeout time.Duration + // timeout for identity event and account event processing (total, including all setup, rules, and teardown) + IdentityEventTimeout time.Duration + // timeout for event processing (total, including all setup, rules, and teardown) + OzoneEventTimeout time.Duration +} + +// Entrypoint for external code pushing #identity events in to the engine. +// +// This method can be called concurrently, though cached state may end up inconsistent if multiple events for the same account (DID) are processed in parallel. +func (eng *Engine) ProcessIdentityEvent(ctx context.Context, evt comatproto.SyncSubscribeRepos_Identity) error { + eventProcessCount.WithLabelValues("identity").Inc() + start := time.Now() + defer func() { + duration := time.Since(start) + eventProcessDuration.WithLabelValues("identity").Observe(duration.Seconds()) + }() + + did, err := syntax.ParseDID(evt.Did) + if err != nil { + return fmt.Errorf("bad DID in repo #identity event (%s): %w", evt.Did, err) + } + + // similar to an HTTP server, we want to recover any panics from rule execution + defer func() { + if r := recover(); r != nil { + eng.Logger.Error("automod event execution exception", "err", r, "did", did, "type", "identity") + eventErrorCount.WithLabelValues("identity").Inc() + } + }() + var cancel context.CancelFunc + if eng.Config.IdentityEventTimeout != 0 { + ctx, cancel = context.WithTimeout(ctx, eng.Config.IdentityEventTimeout) + defer cancel() + } + + // first purge any caches; we need to re-resolve from scratch on identity updates + if err := eng.PurgeAccountCaches(ctx, did); err != nil { + eng.Logger.Error("failed to purge identity cache; identity rule may not run correctly", "err", err) + } + // TODO(bnewbold): if it was a tombstone, this might fail + ident, err := eng.Directory.LookupDID(ctx, did) + if err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("resolving identity: %w", err) + } + if ident == nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("identity not found for DID: %s", did.String()) + } + + var am *AccountMeta + if !eng.Config.SkipAccountMeta { + am, err = eng.GetAccountMeta(ctx, ident) + if err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("failed to fetch account metadata: %w", err) + } + } else { + am = &AccountMeta{ + Identity: ident, + Profile: ProfileSummary{}, + } + } + ac := NewAccountContext(ctx, eng, *am) + if err := eng.Rules.CallIdentityRules(&ac); err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("rule execution failed: %w", err) + } + eng.CanonicalLogLineAccount(&ac) + if err := eng.persistAccountModActions(&ac); err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("failed to persist actions for identity event: %w", err) + } + if err := eng.persistCounters(ctx, ac.effects); err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("failed to persist counters for identity event: %w", err) + } + return nil +} + +// Entrypoint for external code pushing #account events in to the engine. +// +// This method can be called concurrently, though cached state may end up inconsistent if multiple events for the same account (DID) are processed in parallel. +func (eng *Engine) ProcessAccountEvent(ctx context.Context, evt comatproto.SyncSubscribeRepos_Account) error { + eventProcessCount.WithLabelValues("account").Inc() + start := time.Now() + defer func() { + duration := time.Since(start) + eventProcessDuration.WithLabelValues("account").Observe(duration.Seconds()) + }() + + did, err := syntax.ParseDID(evt.Did) + if err != nil { + return fmt.Errorf("bad DID in repo #account event (%s): %w", evt.Did, err) + } + + // similar to an HTTP server, we want to recover any panics from rule execution + defer func() { + if r := recover(); r != nil { + eng.Logger.Error("automod event execution exception", "err", r, "did", did, "type", "account") + eventErrorCount.WithLabelValues("account").Inc() + } + }() + var cancel context.CancelFunc + if eng.Config.IdentityEventTimeout != 0 { + ctx, cancel = context.WithTimeout(ctx, eng.Config.IdentityEventTimeout) + defer cancel() + } + + // first purge any caches; we need to re-resolve from scratch on account updates + if err := eng.PurgeAccountCaches(ctx, did); err != nil { + eng.Logger.Error("failed to purge account cache; account rule may not run correctly", "err", err) + } + // TODO(bnewbold): if it was a tombstone, this might fail + ident, err := eng.Directory.LookupDID(ctx, did) + if err != nil { + eventErrorCount.WithLabelValues("account").Inc() + return fmt.Errorf("resolving identity: %w", err) + } + if ident == nil { + eventErrorCount.WithLabelValues("account").Inc() + return fmt.Errorf("identity not found for DID: %s", did.String()) + } + + var am *AccountMeta + if !eng.Config.SkipAccountMeta { + am, err = eng.GetAccountMeta(ctx, ident) + if err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("failed to fetch account metadata: %w", err) + } + } else { + am = &AccountMeta{ + Identity: ident, + Profile: ProfileSummary{}, + } + } + ac := NewAccountContext(ctx, eng, *am) + if err := eng.Rules.CallAccountRules(&ac); err != nil { + eventErrorCount.WithLabelValues("account").Inc() + return fmt.Errorf("rule execution failed: %w", err) + } + eng.CanonicalLogLineAccount(&ac) + if err := eng.persistAccountModActions(&ac); err != nil { + eventErrorCount.WithLabelValues("account").Inc() + return fmt.Errorf("failed to persist actions for account event: %w", err) + } + if err := eng.persistCounters(ctx, ac.effects); err != nil { + eventErrorCount.WithLabelValues("account").Inc() + return fmt.Errorf("failed to persist counters for account event: %w", err) + } + return nil +} + +// Entrypoint for external code pushing repository updates. A simple repo commit results in multiple calls. +// +// This method can be called concurrently, though cached state may end up inconsistent if multiple events for the same account (DID) are processed in parallel. +func (eng *Engine) ProcessRecordOp(ctx context.Context, op RecordOp) error { + eventProcessCount.WithLabelValues("record").Inc() + start := time.Now() + defer func() { + duration := time.Since(start) + eventProcessDuration.WithLabelValues("record").Observe(duration.Seconds()) + }() + + // similar to an HTTP server, we want to recover any panics from rule execution + defer func() { + if r := recover(); r != nil { + eng.Logger.Error("automod event execution exception", "err", r, "did", op.DID, "collection", op.Collection, "rkey", op.RecordKey) + } + }() + var cancel context.CancelFunc + if eng.Config.RecordEventTimeout != 0 { + ctx, cancel = context.WithTimeout(ctx, eng.Config.RecordEventTimeout) + defer cancel() + } + + if err := op.Validate(); err != nil { + eventErrorCount.WithLabelValues("record").Inc() + return fmt.Errorf("bad record op: %w", err) + } + ident, err := eng.Directory.LookupDID(ctx, op.DID) + if err != nil { + eventErrorCount.WithLabelValues("record").Inc() + return fmt.Errorf("resolving identity: %w", err) + } + if ident == nil { + eventErrorCount.WithLabelValues("record").Inc() + return fmt.Errorf("identity not found for DID: %s", op.DID) + } + + var am *AccountMeta + if !eng.Config.SkipAccountMeta { + am, err = eng.GetAccountMeta(ctx, ident) + if err != nil { + eventErrorCount.WithLabelValues("identity").Inc() + return fmt.Errorf("failed to fetch account metadata: %w", err) + } + } else { + am = &AccountMeta{ + Identity: ident, + Profile: ProfileSummary{}, + } + } + rc := NewRecordContext(ctx, eng, *am, op) + rc.Logger.Debug("processing record") + switch op.Action { + case CreateOp, UpdateOp: + if err := eng.Rules.CallRecordRules(&rc); err != nil { + eventErrorCount.WithLabelValues("record").Inc() + return fmt.Errorf("rule execution failed: %w", err) + } + case DeleteOp: + if err := eng.Rules.CallRecordDeleteRules(&rc); err != nil { + eventErrorCount.WithLabelValues("record").Inc() + return fmt.Errorf("rule execution failed: %w", err) + } + default: + eventErrorCount.WithLabelValues("record").Inc() + return fmt.Errorf("unexpected op action: %s", op.Action) + } + eng.CanonicalLogLineRecord(&rc) + // purge the account meta cache when profile is updated + if rc.RecordOp.Collection == "app.bsky.actor.profile" { + if err := eng.PurgeAccountCaches(ctx, op.DID); err != nil { + eng.Logger.Error("failed to purge identity cache", "err", err) + } + } + if err := eng.persistRecordModActions(&rc); err != nil { + eventErrorCount.WithLabelValues("record").Inc() + return fmt.Errorf("failed to persist actions for record event: %w", err) + } + if err := eng.persistCounters(ctx, rc.effects); err != nil { + eventErrorCount.WithLabelValues("record").Inc() + return fmt.Errorf("failed to persist counts for record event: %w", err) + } + return nil +} + +// Purge metadata caches for a specific account. +func (e *Engine) PurgeAccountCaches(ctx context.Context, did syntax.DID) error { + e.Logger.Debug("purging account caches", "did", did.String()) + dirErr := e.Directory.Purge(ctx, did.AtIdentifier()) + cacheErr := e.Cache.Purge(ctx, "acct", did.String()) + if dirErr != nil { + return dirErr + } + return cacheErr +} + +func (e *Engine) CanonicalLogLineAccount(c *AccountContext) { + c.Logger.Info("canonical-event-line", + "accountLabels", c.effects.AccountLabels, + "accountFlags", c.effects.AccountFlags, + "accountTags", c.effects.AccountTags, + "accountTakedown", c.effects.AccountTakedown, + "accountReports", len(c.effects.AccountReports), + ) +} + +func (e *Engine) CanonicalLogLineRecord(c *RecordContext) { + c.Logger.Info("canonical-event-line", + "accountLabels", c.effects.AccountLabels, + "accountFlags", c.effects.AccountFlags, + "accountTags", c.effects.AccountTags, + "accountTakedown", c.effects.AccountTakedown, + "accountReports", len(c.effects.AccountReports), + "recordLabels", c.effects.RecordLabels, + "recordFlags", c.effects.RecordFlags, + "recordTags", c.effects.RecordTags, + "recordTakedown", c.effects.RecordTakedown, + "recordReports", len(c.effects.RecordReports), + ) +} diff --git a/automod/engine/engine_ozone.go b/automod/engine/engine_ozone.go new file mode 100644 index 000000000..d52c8bf53 --- /dev/null +++ b/automod/engine/engine_ozone.go @@ -0,0 +1,223 @@ +package engine + +import ( + "context" + "fmt" + "time" + + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func NewOzoneEventContext(ctx context.Context, eng *Engine, eventView *toolsozone.ModerationDefs_ModEventView) (*OzoneEventContext, error) { + + if eventView.Event == nil { + return nil, fmt.Errorf("nil ozone event type") + } + + eventType := "" + if eventView.Event.ModerationDefs_ModEventTakedown != nil { + eventType = "takedown" + } else if eventView.Event.ModerationDefs_ModEventReverseTakedown != nil { + eventType = "reverseTakedown" + } else if eventView.Event.ModerationDefs_ModEventComment != nil { + eventType = "comment" + } else if eventView.Event.ModerationDefs_ModEventReport != nil { + eventType = "report" + } else if eventView.Event.ModerationDefs_ModEventLabel != nil { + eventType = "label" + } else if eventView.Event.ModerationDefs_ModEventAcknowledge != nil { + eventType = "acknowledge" + } else if eventView.Event.ModerationDefs_ModEventEscalate != nil { + eventType = "escalate" + } else if eventView.Event.ModerationDefs_ModEventMute != nil { + eventType = "mute" + } else if eventView.Event.ModerationDefs_ModEventUnmute != nil { + eventType = "unmute" + } else if eventView.Event.ModerationDefs_ModEventMuteReporter != nil { + eventType = "muteReporter" + } else if eventView.Event.ModerationDefs_ModEventUnmuteReporter != nil { + eventType = "unmuteReporter" + } else if eventView.Event.ModerationDefs_ModEventEmail != nil { + eventType = "email" + } else if eventView.Event.ModerationDefs_ModEventResolveAppeal != nil { + eventType = "resolveAppeal" + } else if eventView.Event.ModerationDefs_ModEventDivert != nil { + eventType = "divert" + } else if eventView.Event.ModerationDefs_ModEventTag != nil { + eventType = "tag" + } else if eventView.Event.ModerationDefs_ModEventPriorityScore != nil { + eventType = "priorityScore" + } else { + return nil, fmt.Errorf("unhandled ozone event type") + } + + creatorDID, err := syntax.ParseDID(eventView.CreatedBy) + if err != nil { + return nil, err + } + + var subjectDID syntax.DID + var subjectURI *syntax.ATURI + var recordMeta *RecordMeta + if eventView.Subject == nil { + return nil, fmt.Errorf("empty ozone event subject") + } else if eventView.Subject.AdminDefs_RepoRef != nil { + subjectDID, err = syntax.ParseDID(eventView.Subject.AdminDefs_RepoRef.Did) + if err != nil { + return nil, err + } + } else if eventView.Subject.RepoStrongRef != nil { + u, err := syntax.ParseATURI(eventView.Subject.RepoStrongRef.Uri) + if err != nil { + return nil, err + } + subjectURI := &u + subjectDID, err = subjectURI.Authority().AsDID() + if err != nil { + return nil, err + } + cidVal, err := syntax.ParseCID(eventView.Subject.RepoStrongRef.Cid) + if err != nil { + return nil, err + } + recordMeta = &RecordMeta{ + DID: subjectDID, + Collection: subjectURI.Collection(), + RecordKey: subjectURI.RecordKey(), + CID: &cidVal, + } + } else { + return nil, fmt.Errorf("empty ozone event subject") + } + + createdAt, err := syntax.ParseDatetime(eventView.CreatedAt) + if err != nil { + return nil, err + } + + evt := OzoneEvent{ + EventType: eventType, + EventID: eventView.Id, + CreatedAt: createdAt, + CreatedBy: creatorDID, + SubjectDID: subjectDID, + SubjectURI: subjectURI, + // TODO: SubjectBlobs []syntax.CID + Event: *eventView.Event, + } + + creatorIdent, err := eng.Directory.LookupDID(ctx, evt.CreatedBy) + if err != nil { + return nil, err + } + if creatorIdent == nil { + return nil, fmt.Errorf("identity not found for creator DID: %s", evt.CreatedBy) + } + creatorMeta, err := eng.GetAccountMeta(ctx, creatorIdent) + if err != nil { + return nil, err + } + + subjectIdent, err := eng.Directory.LookupDID(ctx, evt.SubjectDID) + if err != nil { + return nil, err + } + if subjectIdent == nil { + return nil, fmt.Errorf("identity not found for subject DID: %s", evt.SubjectDID) + } + accountMeta, err := eng.GetAccountMeta(ctx, subjectIdent) + if err != nil { + return nil, err + } + + return &OzoneEventContext{ + AccountContext: AccountContext{ + BaseContext: BaseContext{ + Ctx: ctx, + Err: nil, + Logger: eng.Logger.With("eventID", evt.EventID, "ozoneEventType", evt.EventType, "creatorDID", evt.CreatedBy, "subjectDID", evt.SubjectDID), + engine: eng, + effects: &Effects{}, + }, + Account: *accountMeta, + }, + Event: evt, + CreatorAccount: *creatorMeta, + SubjectRecord: recordMeta, + }, nil +} + +// Entrypoint for external code pushing ozone events. +// +// This method can be called concurrently, though cached state may end up inconsistent if multiple events for the same account (DID) are processed in parallel. +func (eng *Engine) ProcessOzoneEvent(ctx context.Context, eventView *toolsozone.ModerationDefs_ModEventView) error { + eventProcessCount.WithLabelValues("ozone").Inc() + start := time.Now() + defer func() { + duration := time.Since(start) + eventProcessDuration.WithLabelValues("ozone").Observe(duration.Seconds()) + }() + + // similar to an HTTP server, we want to recover any panics from rule execution + defer func() { + if r := recover(); r != nil { + eng.Logger.Error("automod ozone event execution exception", "err", r, "eventID", eventView.Id, "createdAt", eventView.CreatedAt) + } + }() + var cancel context.CancelFunc + if eng.Config.OzoneEventTimeout != 0 { + ctx, cancel = context.WithTimeout(ctx, eng.Config.OzoneEventTimeout) + defer cancel() + } + + ec, err := NewOzoneEventContext(ctx, eng, eventView) + if err != nil { + eventErrorCount.WithLabelValues("ozoneEvent").Inc() + return fmt.Errorf("failed to hydrate ozone event context: %w", err) + } + + // if this is a "self-event", created by automod itself, skip it to prevent a loop + if ec.Event.CreatedBy.String() == eng.OzoneClient.Auth.Did { + ec.Logger.Debug("skipping ozone self-event") + return nil + } + + ec.Logger.Debug("processing ozone event") + + if err := eng.Rules.CallOzoneEventRules(ec); err != nil { + eventErrorCount.WithLabelValues("ozoneEvent").Inc() + return fmt.Errorf("ozone rule execution failed: %w", err) + } + + eng.CanonicalLogLineOzoneEvent(ec) + + // some ozone events should result in account meta cache flushes + if (ec.Event.EventType == "takedown" || ec.Event.EventType == "reverseTakedown" || ec.Event.EventType == "label" || ec.Event.EventType == "tag") && ec.SubjectRecord == nil { + if err := eng.PurgeAccountCaches(ctx, ec.Event.SubjectDID); err != nil { + eng.Logger.Error("failed to purge identity cache", "err", err, "did", ec.Event.SubjectDID) + } + } + if err := eng.persistAccountModActions(&ec.AccountContext); err != nil { + eventErrorCount.WithLabelValues("ozoneEvent").Inc() + return fmt.Errorf("failed to persist actions for ozone event: %w", err) + } + if err := eng.persistCounters(ctx, ec.effects); err != nil { + eventErrorCount.WithLabelValues("ozoneEvent").Inc() + return fmt.Errorf("failed to persist counts for ozone event: %w", err) + } + return nil +} + +func (e *Engine) CanonicalLogLineOzoneEvent(c *OzoneEventContext) { + c.Logger.Info("canonical-event-line", + "accountLabels", c.effects.AccountLabels, + "accountFlags", c.effects.AccountFlags, + "accountTakedown", c.effects.AccountTakedown, + "accountReports", len(c.effects.AccountReports), + "recordLabels", c.effects.RecordLabels, + "recordFlags", c.effects.RecordFlags, + "recordTakedown", c.effects.RecordTakedown, + "recordReports", len(c.effects.RecordReports), + ) +} diff --git a/automod/engine/engine_test.go b/automod/engine/engine_test.go new file mode 100644 index 000000000..0cbf30352 --- /dev/null +++ b/automod/engine/engine_test.go @@ -0,0 +1,51 @@ +package engine + +import ( + "bytes" + "context" + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/stretchr/testify/assert" +) + +func TestEngineBasics(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + eng := EngineTestFixture() + id1 := identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + } + cid1 := syntax.CID("cid123") + p1 := appbsky.FeedPost{ + Text: "some post blah", + } + p1buf := new(bytes.Buffer) + assert.NoError(p1.MarshalCBOR(p1buf)) + p1cbor := p1buf.Bytes() + + op := RecordOp{ + Action: CreateOp, + DID: id1.DID, + Collection: syntax.NSID("app.bsky.feed.post"), + RecordKey: syntax.RecordKey("abc123"), + CID: &cid1, + RecordCBOR: p1cbor, + } + assert.NoError(eng.ProcessRecordOp(ctx, op)) + + p2 := appbsky.FeedPost{ + Text: "some post blah", + Tags: []string{"one", "slur"}, + } + p2buf := new(bytes.Buffer) + assert.NoError(p2.MarshalCBOR(p2buf)) + p2cbor := p2buf.Bytes() + op.RecordCBOR = p2cbor + assert.NoError(eng.ProcessRecordOp(ctx, op)) +} diff --git a/automod/engine/fetch_account_meta.go b/automod/engine/fetch_account_meta.go new file mode 100644 index 000000000..39b4df1eb --- /dev/null +++ b/automod/engine/fetch_account_meta.go @@ -0,0 +1,203 @@ +package engine + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/xrpc" +) + +var newAccountRetryDuration = 3 * 1000 * time.Millisecond + +// Helper to hydrate metadata about an account from several sources: PDS (if access), mod service (if access), public identity resolution +func (e *Engine) GetAccountMeta(ctx context.Context, ident *identity.Identity) (*AccountMeta, error) { + + logger := e.Logger.With("did", ident.DID.String()) + + // fallback in case client wasn't configured (eg, testing) + if e.BskyClient == nil { + logger.Debug("skipping account meta hydration") + am := AccountMeta{ + Identity: ident, + Profile: ProfileSummary{}, + } + return &am, nil + } + + existing, err := e.Cache.Get(ctx, "acct", ident.DID.String()) + if err != nil { + return nil, fmt.Errorf("failed checking account meta cache: %w", err) + } + if existing != "" { + var am AccountMeta + err := json.Unmarshal([]byte(existing), &am) + if err != nil { + return nil, fmt.Errorf("parsing AccountMeta from cache: %v", err) + } + am.Identity = ident + return &am, nil + } + + // doing a "full" fetch from here on + accountMetaFetches.Inc() + am := AccountMeta{ + Identity: ident, + } + + // automod-internal "flags" + flags, err := e.Flags.Get(ctx, ident.DID.String()) + if err != nil { + return nil, fmt.Errorf("failed checking account flag cache: %w", err) + } + am.AccountFlags = flags + + // fetch account metadata from AppView + pv, err := appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String()) + // most common cause of this is a race between automod and ozone/appview for new accounts. just sleep a couple seconds and retry! + var xrpcError *xrpc.Error + if err != nil && errors.As(err, &xrpcError) && (xrpcError.StatusCode == 400 || xrpcError.StatusCode == 404) { + logger.Debug("account profile lookup initially failed (from bsky appview), will retry", "err", err, "sleepDuration", newAccountRetryDuration) + time.Sleep(newAccountRetryDuration) + pv, err = appbsky.ActorGetProfile(ctx, e.BskyClient, ident.DID.String()) + } + if err != nil { + logger.Warn("account profile lookup failed (from bsky appview)", "err", err) + return &am, nil + } + + am.Profile = ProfileSummary{ + HasAvatar: pv.Avatar != nil, + AvatarCid: cidFromCdnUrl(pv.Avatar), + BannerCid: cidFromCdnUrl(pv.Banner), + Description: pv.Description, + DisplayName: pv.DisplayName, + } + if pv.PostsCount != nil { + am.PostsCount = *pv.PostsCount + } + if pv.FollowersCount != nil { + am.FollowersCount = *pv.FollowersCount + } + if pv.FollowsCount != nil { + am.FollowsCount = *pv.FollowsCount + } + + var labels []string + var negLabels []string + for _, lbl := range pv.Labels { + if lbl.Neg != nil && *lbl.Neg == true { + negLabels = append(negLabels, lbl.Val) + } else { + labels = append(labels, lbl.Val) + } + } + am.AccountLabels = dedupeStrings(labels) + am.AccountNegatedLabels = dedupeStrings(negLabels) + + if pv.CreatedAt != nil { + ts, err := syntax.ParseDatetimeTime(*pv.CreatedAt) + if err != nil { + logger.Warn("invalid profile createdAt", "err", err, "createdAt", pv.CreatedAt) + } else { + am.CreatedAt = &ts + } + } + + // first attempt to fetch private account metadata from Ozone + if e.OzoneClient != nil { + rd, err := toolsozone.ModerationGetRepo(ctx, e.OzoneClient, ident.DID.String()) + if err != nil { + logger.Warn("failed to fetch private account metadata from Ozone", "err", err) + } else { + ap := AccountPrivate{} + if rd.Email != nil && *rd.Email != "" { + ap.Email = *rd.Email + } + if rd.EmailConfirmedAt != nil && *rd.EmailConfirmedAt != "" { + ap.EmailConfirmed = true + } + // TODO: ozone doesn't really return good account "created at", just just leave that field nil + ap.IndexedAt = nil + if rd.DeactivatedAt != nil { + am.Deactivated = true + } + if rd.Moderation != nil && rd.Moderation.SubjectStatus != nil { + if rd.Moderation.SubjectStatus.Takendown != nil && *rd.Moderation.SubjectStatus.Takendown == true { + am.Takendown = true + } + if rd.Moderation.SubjectStatus.Appealed != nil && *rd.Moderation.SubjectStatus.Appealed == true { + ap.Appealed = true + } + ap.AccountTags = dedupeStrings(rd.Moderation.SubjectStatus.Tags) + if rd.Moderation.SubjectStatus.ReviewState != nil { + switch *rd.Moderation.SubjectStatus.ReviewState { + case "tools.ozone.moderation.defs#reviewOpen": + ap.ReviewState = ReviewStateOpen + case "tools.ozone.moderation.defs#reviewEscalated": + ap.ReviewState = ReviewStateEscalated + case "tools.ozone.moderation.defs#reviewClosed": + ap.ReviewState = ReviewStateClosed + case "tools.ozone.moderation.defs#reviewNone": + ap.ReviewState = ReviewStateNone + default: + logger.Warn("unexpected ozone moderation review state", "state", rd.Moderation.SubjectStatus.ReviewState, "did", ident.DID) + } + } + } + if rd.ThreatSignatures != nil || len(rd.ThreatSignatures) > 0 { + asigs := make([]AbuseSignature, len(rd.ThreatSignatures)) + for i, sig := range rd.ThreatSignatures { + asigs[i] = AbuseSignature{Property: sig.Property, Value: sig.Value} + } + ap.AbuseSignatures = asigs + } + am.Private = &ap + } + } + // fall back to PDS/entryway fetching; less metadata available + if am.Private == nil && e.AdminClient != nil { + pv, err := comatproto.AdminGetAccountInfo(ctx, e.AdminClient, ident.DID.String()) + if err != nil { + logger.Warn("failed to fetch private account metadata from PDS/entryway", "err", err) + } else { + ap := AccountPrivate{} + if pv.Email != nil && *pv.Email != "" { + ap.Email = *pv.Email + } + if pv.EmailConfirmedAt != nil && *pv.EmailConfirmedAt != "" { + ap.EmailConfirmed = true + } + ts, err := syntax.ParseDatetimeTime(pv.IndexedAt) + if err != nil { + return nil, fmt.Errorf("bad entryway account IndexedAt: %w", err) + } + ap.IndexedAt = &ts + am.Private = &ap + if am.CreatedAt == nil { + am.CreatedAt = &ts + } + } + } + + if am.CreatedAt == nil { + logger.Warn("account metadata missing CreatedAt time") + } + + val, err := json.Marshal(&am) + if err != nil { + return nil, err + } + + if err := e.Cache.Set(ctx, "acct", ident.DID.String(), string(val)); err != nil { + logger.Error("writing to account meta cache failed", "err", err) + } + return &am, nil +} diff --git a/automod/engine/fetch_relationship.go b/automod/engine/fetch_relationship.go new file mode 100644 index 000000000..50226830d --- /dev/null +++ b/automod/engine/fetch_relationship.go @@ -0,0 +1,67 @@ +package engine + +import ( + "context" + "encoding/json" + "fmt" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// Represents app.bsky social graph relationship between a primary account, and an "other" account +type AccountRelationship struct { + // the DID of the "other" account + DID syntax.DID + // if the primary account is followed by this "other" DID + FollowedBy bool + // if the primary account follows this "other" DID + Following bool +} + +// Helper to fetch social graph relationship metadata +func (eng *Engine) GetAccountRelationship(ctx context.Context, primary, other syntax.DID) (*AccountRelationship, error) { + + logger := eng.Logger.With("did", primary.String(), "otherDID", other.String()) + + cacheKey := fmt.Sprintf("%s/%s", primary.String(), other.String()) + existing, err := eng.Cache.Get(ctx, "graph-rel", cacheKey) + if err != nil { + return nil, fmt.Errorf("failed checking account relationship cache: %w", err) + } + if existing != "" { + var ar AccountRelationship + err := json.Unmarshal([]byte(existing), &ar) + if err != nil || ar.DID != primary { + return nil, fmt.Errorf("parsing AccountRelationship from cache: %v", err) + } + return &ar, nil + } + + if eng.BskyClient == nil { + logger.Warn("skipping account relationship fetch") + ar := AccountRelationship{DID: other} + return &ar, nil + } + + // fetch account relationship from AppView + accountRelationshipFetches.Inc() + resp, err := appbsky.GraphGetRelationships(ctx, eng.BskyClient, primary.String(), []string{other.String()}) + if err != nil || len(resp.Relationships) != 1 { + logger.Warn("account relationship lookup failed", "err", err) + ar := AccountRelationship{DID: other} + return &ar, nil + } + + if len(resp.Relationships) != 1 || resp.Relationships[0].GraphDefs_Relationship == nil { + logger.Warn("account relationship actor not found") + } + + rel := resp.Relationships[0].GraphDefs_Relationship + ar := AccountRelationship{ + DID: primary, + FollowedBy: rel.FollowedBy != nil, + Following: rel.Following != nil, + } + return &ar, nil +} diff --git a/automod/engine/metrics.go b/automod/engine/metrics.go new file mode 100644 index 000000000..bc32b8e54 --- /dev/null +++ b/automod/engine/metrics.go @@ -0,0 +1,76 @@ +package engine + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var eventProcessDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "automod_event_duration_sec", + Help: "Total duration of automod event processing", +}, []string{"type"}) + +var eventProcessCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_event_processed", + Help: "Number of events processed", +}, []string{"type"}) + +var eventErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_event_errors", + Help: "Number of events which failed processing", +}, []string{"type"}) + +var actionNewLabelCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_labels", + Help: "Number of new labels persisted", +}, []string{"type", "val"}) + +var actionNewTagCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_tags", + Help: "Number of new tags persisted", +}, []string{"type", "val"}) + +var actionNewFlagCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_flags", + Help: "Number of new flags persisted", +}, []string{"type", "val"}) + +var actionNewReportCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_reports", + Help: "Number of new flags persisted", +}, []string{"type"}) + +var actionNewTakedownCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_takedowns", + Help: "Number of new takedowns", +}, []string{"type"}) + +var actionNewEscalationCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_escalations", + Help: "Number of new subject escalations", +}, []string{"type"}) + +var actionNewAcknowledgeCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_new_action_acknowledges", + Help: "Number of new subjects acknowledged", +}, []string{"type"}) + +var accountMetaFetches = promauto.NewCounter(prometheus.CounterOpts{ + Name: "automod_account_meta_fetches", + Help: "Number of account metadata reads (API calls)", +}) + +var accountRelationshipFetches = promauto.NewCounter(prometheus.CounterOpts{ + Name: "automod_account_relationship_fetches", + Help: "Number of account relationship reads (API calls)", +}) + +var blobDownloadCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_blob_downloads", + Help: "Number of blobs downloaded, by HTTP status code", +}, []string{"status"}) + +var blobDownloadDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "automod_blob_download_duration_sec", + Help: "Duration of blob download attempts", +}) diff --git a/automod/engine/notifier.go b/automod/engine/notifier.go new file mode 100644 index 000000000..f04c0b76b --- /dev/null +++ b/automod/engine/notifier.go @@ -0,0 +1,11 @@ +package engine + +import ( + "context" +) + +// Interface for a type that can handle sending notifications +type Notifier interface { + SendAccount(ctx context.Context, service string, c *AccountContext) error + SendRecord(ctx context.Context, service string, c *RecordContext) error +} diff --git a/automod/engine/persist.go b/automod/engine/persist.go new file mode 100644 index 000000000..8ac3f18b8 --- /dev/null +++ b/automod/engine/persist.go @@ -0,0 +1,514 @@ +package engine + +import ( + "context" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/automod/keyword" +) + +func (eng *Engine) persistCounters(ctx context.Context, eff *Effects) error { + // TODO: dedupe this array + for _, ref := range eff.CounterIncrements { + if ref.Period != nil { + err := eng.Counters.IncrementPeriod(ctx, ref.Name, ref.Val, *ref.Period) + if err != nil { + return err + } + } else { + err := eng.Counters.Increment(ctx, ref.Name, ref.Val) + if err != nil { + return err + } + } + } + for _, ref := range eff.CounterDistinctIncrements { + err := eng.Counters.IncrementDistinct(ctx, ref.Name, ref.Bucket, ref.Val) + if err != nil { + return err + } + } + return nil +} + +// Persists account-level moderation actions: new labels, new tags, new flags, new takedowns, and reports. +// +// If necessary, will "purge" identity and account caches, so that state updates will be picked up for subsequent events. +// +// Note that this method expects to run *before* counts are persisted (it accesses and updates some counts) +func (eng *Engine) persistAccountModActions(c *AccountContext) error { + ctx := c.Ctx + + // de-dupe actions + newLabels := dedupeLabelActions(c.effects.AccountLabels, c.Account.AccountLabels, c.Account.AccountNegatedLabels) + rmdLabels := []string{} + for _, lbl := range dedupeStrings(c.effects.RemovedAccountLabels) { + // we don't need to try and remove labels whenever they are either _not_ already in the account labels, _or_ if they are + // being applied by some other rule before persisting + if !keyword.TokenInSet(lbl, c.Account.AccountLabels) || keyword.TokenInSet(lbl, c.effects.AccountLabels) { + continue + } + rmdLabels = append(rmdLabels, lbl) + } + existingTags := []string{} + if c.Account.Private != nil { + existingTags = c.Account.Private.AccountTags + } + newTags := dedupeTagActions(c.effects.AccountTags, existingTags) + newFlags := dedupeFlagActions(c.effects.AccountFlags, c.Account.AccountFlags) + + // don't report the same account multiple times on the same day for the same reason. this is a quick check; we also query the mod service API just before creating the report. + partialReports, err := eng.dedupeReportActions(ctx, c.Account.Identity.DID.String(), c.effects.AccountReports) + if err != nil { + return fmt.Errorf("de-duplicating reports: %w", err) + } + newReports, err := eng.circuitBreakReports(ctx, partialReports) + if err != nil { + return fmt.Errorf("circuit-breaking reports: %w", err) + } + newTakedown, err := eng.circuitBreakTakedown(ctx, c.effects.AccountTakedown && !c.Account.Takendown) + if err != nil { + return fmt.Errorf("circuit-breaking takedowns: %w", err) + } + newEscalation := c.effects.AccountEscalate + if c.Account.Private != nil && c.Account.Private.ReviewState == ReviewStateEscalated { + // de-dupe account escalation + newEscalation = false + } else { + newEscalation, err = eng.circuitBreakModAction(ctx, newEscalation) + if err != nil { + return fmt.Errorf("circuit-breaking escalation: %w", err) + } + } + newAcknowledge := c.effects.AccountAcknowledge + if c.Account.Private != nil && (c.Account.Private.ReviewState == "closed" || c.Account.Private.ReviewState == "none") { + // de-dupe account escalation + newAcknowledge = false + } else { + newAcknowledge, err = eng.circuitBreakModAction(ctx, newAcknowledge) + if err != nil { + return fmt.Errorf("circuit-breaking acknowledge: %w", err) + } + } + + anyModActions := newTakedown || newEscalation || newAcknowledge || len(newLabels) > 0 || len(rmdLabels) > 0 || len(newTags) > 0 || len(newFlags) > 0 || len(newReports) > 0 + if anyModActions && eng.Notifier != nil { + for _, srv := range dedupeStrings(c.effects.NotifyServices) { + if err := eng.Notifier.SendAccount(ctx, srv, c); err != nil { + c.Logger.Error("failed to deliver notification", "service", srv, "err", err) + } + } + } + + // flags don't require admin auth + if len(newFlags) > 0 { + for _, val := range newFlags { + // note: WithLabelValues is a prometheus label, not an atproto label + actionNewFlagCount.WithLabelValues("record", val).Inc() + } + eng.Flags.Add(ctx, c.Account.Identity.DID.String(), newFlags) + } + + // if we can't actually talk to service, bail out early + if eng.OzoneClient == nil { + if anyModActions { + c.Logger.Warn("not persisting actions, mod service client not configured") + } + return nil + } + + xrpcc := eng.OzoneClient + + if len(newLabels) > 0 || len(rmdLabels) > 0 { + c.Logger.Info("updating account labels", "newLabels", newLabels, "rmdLabels", rmdLabels) + for _, val := range newLabels { + // note: WithLabelValues is a prometheus label, not an atproto label + actionNewLabelCount.WithLabelValues("account", val).Inc() + } + comment := "[automod]: auto-labeling account" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventLabel: &toolsozone.ModerationDefs_ModEventLabel{ + CreateLabelVals: newLabels, + NegateLabelVals: rmdLabels, + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to create account labels", "err", err) + } + } + + if len(newTags) > 0 { + c.Logger.Info("tagging account", "newTags", newTags) + for _, val := range newTags { + // note: WithLabelValues is a prometheus label, not an atproto label + actionNewTagCount.WithLabelValues("account", val).Inc() + } + comment := "[automod]: auto-tagging account" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventTag: &toolsozone.ModerationDefs_ModEventTag{ + Add: newTags, + Remove: []string{}, + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to create account tags", "err", err) + } + } + + // reports are additionally de-duped when persisting the action, so track with a flag + createdReports := false + for _, mr := range newReports { + created, err := eng.createReportIfFresh(ctx, xrpcc, c.Account.Identity.DID, mr) + if err != nil { + c.Logger.Error("failed to create account report", "err", err) + } + if created { + createdReports = true + } + } + + if newTakedown { + c.Logger.Warn("account-takedown") + actionNewTakedownCount.WithLabelValues("account").Inc() + comment := "[automod]: auto account-takedown" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventTakedown: &toolsozone.ModerationDefs_ModEventTakedown{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to execute account takedown", "err", err) + } + + // we don't want to escalate if there is a takedown + newEscalation = false + } + + if newEscalation { + c.Logger.Info("account-escalate") + actionNewEscalationCount.WithLabelValues("account").Inc() + comment := "[automod]: auto account-escalation" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventEscalate: &toolsozone.ModerationDefs_ModEventEscalate{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to execute account escalation", "err", err) + } + } + + if newAcknowledge { + c.Logger.Info("account-acknowledge") + actionNewAcknowledgeCount.WithLabelValues("account").Inc() + comment := "[automod]: auto account-acknowledge" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventAcknowledge: &toolsozone.ModerationDefs_ModEventAcknowledge{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: c.Account.Identity.DID.String(), + }, + }, + }) + if err != nil { + c.Logger.Error("failed to execute account acknowledge", "err", err) + } + } + + needCachePurge := newTakedown || newEscalation || newAcknowledge || len(newLabels) > 0 || len(newTags) > 0 || len(newFlags) > 0 || createdReports + if needCachePurge { + return eng.PurgeAccountCaches(ctx, c.Account.Identity.DID) + } + + return nil +} + +// Persists some record-level state: labels, tags, takedowns, reports. +// +// NOTE: this method currently does *not* persist record-level flags to any storage, and does not de-dupe most actions, on the assumption that the record is new (from firehose) and has no existing mod state. +func (eng *Engine) persistRecordModActions(c *RecordContext) error { + ctx := c.Ctx + if err := eng.persistAccountModActions(&c.AccountContext); err != nil { + return err + } + + atURI := c.RecordOp.ATURI().String() + newLabels := dedupeStrings(c.effects.RecordLabels) + rmdLabels := []string{} + newTags := dedupeStrings(c.effects.RecordTags) + newEscalation := c.effects.RecordEscalate + newAcknowledge := c.effects.RecordAcknowledge + + if (newEscalation || newAcknowledge || len(newLabels) > 0 || len(rmdLabels) > 0 || len(newTags) > 0) && eng.OzoneClient != nil { + // fetch existing record labels, tags, etc + rv, err := toolsozone.ModerationGetRecord(ctx, eng.OzoneClient, c.RecordOp.CID.String(), c.RecordOp.ATURI().String()) + if err != nil { + // NOTE: there is a frequent 4xx error here from Ozone because this record has not been indexed yet + c.Logger.Warn("failed to fetch private record metadata from Ozone", "err", err) + } else { + var existingLabels []string + var negLabels []string + for _, lbl := range rv.Labels { + if lbl.Neg != nil && *lbl.Neg == true { + negLabels = append(negLabels, lbl.Val) + } else { + existingLabels = append(existingLabels, lbl.Val) + } + } + existingLabels = dedupeStrings(existingLabels) + negLabels = dedupeStrings(negLabels) + newLabels = dedupeLabelActions(newLabels, existingLabels, negLabels) + for _, lbl := range dedupeStrings(c.effects.RemovedRecordLabels) { + // we don't need to try and remove labels whenever they are either _not_ already in the record labels, _or_ if they are + // being applied by some other rule before persisting + if !keyword.TokenInSet(lbl, existingLabels) || keyword.TokenInSet(lbl, newLabels) { + continue + } + rmdLabels = append(rmdLabels, lbl) + } + existingTags := []string{} + hasSubjectStatus := rv.Moderation != nil && rv.Moderation.SubjectStatus != nil + if hasSubjectStatus && rv.Moderation.SubjectStatus.Tags != nil { + existingTags = rv.Moderation.SubjectStatus.Tags + } + newTags = dedupeTagActions(newTags, existingTags) + newEscalation = newEscalation && hasSubjectStatus && *rv.Moderation.SubjectStatus.ReviewState != "tools.ozone.moderation.defs#reviewEscalate" + newAcknowledge = newAcknowledge && hasSubjectStatus && *rv.Moderation.SubjectStatus.ReviewState != "tools.ozone.moderation.defs#reviewNone" && *rv.Moderation.SubjectStatus.ReviewState != "tools.ozone.moderation.defs#reviewClosed" + } + } + + newFlags := dedupeStrings(c.effects.RecordFlags) + if len(newFlags) > 0 { + // fetch existing flags, and de-dupe + existingFlags, err := eng.Flags.Get(ctx, atURI) + if err != nil { + return fmt.Errorf("failed checking record flag cache: %w", err) + } + newFlags = dedupeFlagActions(newFlags, existingFlags) + } + + // don't report the same record multiple times on the same day for the same reason. this is a quick check; we also query the mod service API just before creating the report. + partialReports, err := eng.dedupeReportActions(ctx, atURI, c.effects.RecordReports) + if err != nil { + return fmt.Errorf("de-duplicating reports: %w", err) + } + newReports, err := eng.circuitBreakReports(ctx, partialReports) + if err != nil { + return fmt.Errorf("failed to circuit break reports: %w", err) + } + newTakedown, err := eng.circuitBreakTakedown(ctx, c.effects.RecordTakedown) + if err != nil { + return fmt.Errorf("failed to circuit break takedowns: %w", err) + } + newEscalation, err = eng.circuitBreakModAction(ctx, newEscalation) + if err != nil { + return fmt.Errorf("circuit-breaking escalation: %w", err) + } + newAcknowledge, err = eng.circuitBreakModAction(ctx, newAcknowledge) + if err != nil { + return fmt.Errorf("circuit-breaking acknowledge: %w", err) + } + + if newEscalation || newAcknowledge || newTakedown || len(newLabels) > 0 || len(rmdLabels) > 0 || len(newTags) > 0 || len(newFlags) > 0 || len(newReports) > 0 { + if eng.Notifier != nil { + for _, srv := range dedupeStrings(c.effects.NotifyServices) { + if err := eng.Notifier.SendRecord(ctx, srv, c); err != nil { + c.Logger.Error("failed to deliver notification", "service", srv, "err", err) + } + } + } + } + + // flags don't require admin auth + if len(newFlags) > 0 { + for _, val := range newFlags { + // note: WithLabelValues is a prometheus label, not an atproto label + actionNewFlagCount.WithLabelValues("record", val).Inc() + } + eng.Flags.Add(ctx, atURI, newFlags) + } + + // exit early + if !newAcknowledge && !newEscalation && !newTakedown && len(newLabels) == 0 && len(rmdLabels) == 0 && len(newTags) == 0 && len(newReports) == 0 { + return nil + } + + if eng.OzoneClient == nil { + c.Logger.Warn("not persisting actions because mod service client not configured") + return nil + } + + if c.RecordOp.CID == nil { + c.Logger.Warn("skipping record actions because CID is nil, can't construct strong ref") + return nil + } + cid := *c.RecordOp.CID + strongRef := comatproto.RepoStrongRef{ + Cid: cid.String(), + Uri: atURI, + } + + xrpcc := eng.OzoneClient + if len(newLabels) > 0 || len(rmdLabels) > 0 { + c.Logger.Info("updating record labels", "newLabels", newLabels, "rmdLabels", rmdLabels) + for _, val := range newLabels { + // note: WithLabelValues is a prometheus label, not an atproto label + actionNewLabelCount.WithLabelValues("record", val).Inc() + } + comment := "[automod]: auto-labeling record" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventLabel: &toolsozone.ModerationDefs_ModEventLabel{ + CreateLabelVals: newLabels, + NegateLabelVals: rmdLabels, + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + c.Logger.Error("failed to create record label", "err", err) + } + } + + if len(newTags) > 0 { + c.Logger.Info("tagging record", "newTags", newTags) + for _, val := range newTags { + // note: WithLabelValues is a prometheus label, not an atproto label + actionNewTagCount.WithLabelValues("record", val).Inc() + } + comment := "[automod]: auto-tagging record" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventTag: &toolsozone.ModerationDefs_ModEventTag{ + Add: newTags, + Remove: []string{}, + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + c.Logger.Error("failed to create record tag", "err", err) + } + } + + for _, mr := range newReports { + _, err := eng.createRecordReportIfFresh(ctx, xrpcc, c.RecordOp.ATURI(), c.RecordOp.CID, mr) + if err != nil { + c.Logger.Error("failed to create record report", "err", err) + } + } + + if newTakedown { + c.Logger.Warn("record-takedown") + actionNewTakedownCount.WithLabelValues("record").Inc() + comment := "[automod]: automated record-takedown" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventTakedown: &toolsozone.ModerationDefs_ModEventTakedown{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &strongRef, + }, + SubjectBlobCids: dedupeStrings(c.effects.BlobTakedowns), + }) + if err != nil { + c.Logger.Error("failed to execute record takedown", "err", err) + } + + // we don't want to escalate if there is a takedown + newEscalation = false + } + + if newEscalation { + c.Logger.Warn("record-escalation") + actionNewEscalationCount.WithLabelValues("record").Inc() + comment := "[automod]: automated record-escalation" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventEscalate: &toolsozone.ModerationDefs_ModEventEscalate{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + c.Logger.Error("failed to execute record escalation", "err", err) + } + } + + if newAcknowledge { + c.Logger.Warn("record-acknowledge") + actionNewAcknowledgeCount.WithLabelValues("record").Inc() + comment := "[automod]: automated record-acknowledge" + _, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventAcknowledge: &toolsozone.ModerationDefs_ModEventAcknowledge{ + Comment: &comment, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &strongRef, + }, + }) + if err != nil { + c.Logger.Error("failed to execute record acknowledge", "err", err) + } + } + return nil +} diff --git a/automod/engine/persisthelpers.go b/automod/engine/persisthelpers.go new file mode 100644 index 000000000..5642312bf --- /dev/null +++ b/automod/engine/persisthelpers.go @@ -0,0 +1,332 @@ +package engine + +import ( + "context" + "fmt" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/xrpc" +) + +func dedupeLabelActions(labels, existing, existingNegated []string) []string { + newLabels := []string{} + for _, val := range dedupeStrings(labels) { + exists := false + for _, e := range existingNegated { + if val == e { + exists = true + break + } + } + for _, e := range existing { + if val == e { + exists = true + break + } + } + if !exists { + newLabels = append(newLabels, val) + } + } + return newLabels +} + +func dedupeTagActions(tags, existing []string) []string { + newTags := []string{} + for _, val := range dedupeStrings(tags) { + exists := false + for _, e := range existing { + if val == e { + exists = true + break + } + } + if !exists { + newTags = append(newTags, val) + } + } + return newTags +} + +func dedupeFlagActions(flags, existing []string) []string { + newFlags := []string{} + for _, val := range dedupeStrings(flags) { + exists := false + for _, e := range existing { + if val == e { + exists = true + break + } + } + if !exists { + newFlags = append(newFlags, val) + } + } + return newFlags +} + +func (eng *Engine) dedupeReportActions(ctx context.Context, subject string, reports []ModReport) ([]ModReport, error) { + newReports := []ModReport{} + for _, r := range reports { + counterName := "automod-account-report-" + ReasonShortName(r.ReasonType) + existing, err := eng.Counters.GetCount(ctx, counterName, subject, countstore.PeriodDay) + if err != nil { + return nil, fmt.Errorf("checking report de-dupe counts: %w", err) + } + if existing > 0 { + eng.Logger.Debug("skipping account report due to counter", "existing", existing, "reason", ReasonShortName(r.ReasonType)) + } else { + err = eng.Counters.Increment(ctx, counterName, subject) + if err != nil { + return nil, fmt.Errorf("incrementing report de-dupe count: %w", err) + } + newReports = append(newReports, r) + } + } + return newReports, nil +} + +func (eng *Engine) circuitBreakReports(ctx context.Context, reports []ModReport) ([]ModReport, error) { + if len(reports) == 0 { + return []ModReport{}, nil + } + c, err := eng.Counters.GetCount(ctx, "automod-quota", "report", countstore.PeriodDay) + if err != nil { + return nil, fmt.Errorf("checking report action quota: %w", err) + } + + quotaModReportDay := eng.Config.QuotaModReportDay + if quotaModReportDay == 0 { + quotaModReportDay = 10000 + } + if c >= quotaModReportDay { + eng.Logger.Warn("CIRCUIT BREAKER: automod reports") + return []ModReport{}, nil + } + err = eng.Counters.Increment(ctx, "automod-quota", "report") + if err != nil { + return nil, fmt.Errorf("incrementing report action quota: %w", err) + } + return reports, nil +} + +func (eng *Engine) circuitBreakTakedown(ctx context.Context, takedown bool) (bool, error) { + if !takedown { + return false, nil + } + c, err := eng.Counters.GetCount(ctx, "automod-quota", "takedown", countstore.PeriodDay) + if err != nil { + return false, fmt.Errorf("checking takedown action quota: %w", err) + } + quotaModTakedownDay := eng.Config.QuotaModTakedownDay + if quotaModTakedownDay == 0 { + quotaModTakedownDay = 200 + } + if c >= quotaModTakedownDay { + eng.Logger.Warn("CIRCUIT BREAKER: automod takedowns") + return false, nil + } + err = eng.Counters.Increment(ctx, "automod-quota", "takedown") + if err != nil { + return false, fmt.Errorf("incrementing takedown action quota: %w", err) + } + return takedown, nil +} + +// Combined circuit breaker for miscellaneous mod actions like: escalate, acknowledge +func (eng *Engine) circuitBreakModAction(ctx context.Context, action bool) (bool, error) { + if !action { + return false, nil + } + c, err := eng.Counters.GetCount(ctx, "automod-quota", "mod-action", countstore.PeriodDay) + if err != nil { + return false, fmt.Errorf("checking mod action quota: %w", err) + } + quotaModActionDay := eng.Config.QuotaModActionDay + if quotaModActionDay == 0 { + quotaModActionDay = 2000 + } + if c >= quotaModActionDay { + eng.Logger.Warn("CIRCUIT BREAKER: automod action") + return false, nil + } + err = eng.Counters.Increment(ctx, "automod-quota", "mod-action") + if err != nil { + return false, fmt.Errorf("incrementing mod action quota: %w", err) + } + return action, nil +} + +// Creates a moderation report, but checks first if there was a similar recent one, and skips if so. +// +// Returns a bool indicating if a new report was created. +func (eng *Engine) createReportIfFresh(ctx context.Context, xrpcc *xrpc.Client, did syntax.DID, mr ModReport) (bool, error) { + // before creating a report, query to see if automod has already reported this account in the past week for the same reason + // NOTE: this is running in an inner loop (if there are multiple reports), which is a bit inefficient, but seems acceptable + + resp, err := toolsozone.ModerationQueryEvents( + ctx, + xrpcc, + nil, // addedLabels []string + nil, // addedTags []string + "", // ageAssuranceState + "", // batchId string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + xrpcc.Auth.Did, // createdBy string + "", // cursor string + false, // hasComment bool + false, // includeAllUserRecords bool + 5, // limit int64 + nil, // modTool + nil, // policies []string + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + did.String(), // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string + false, // withStrike bool + ) + + if err != nil { + return false, err + } + for _, modEvt := range resp.Events { + // defensively ensure that our query params worked correctly + if modEvt.Event.ModerationDefs_ModEventReport == nil || modEvt.CreatedBy != xrpcc.Auth.Did || modEvt.Subject.AdminDefs_RepoRef == nil || modEvt.Subject.AdminDefs_RepoRef.Did != did.String() || (modEvt.Event.ModerationDefs_ModEventReport.ReportType != nil && *modEvt.Event.ModerationDefs_ModEventReport.ReportType != mr.ReasonType) { + continue + } + // igonre if older + created, err := syntax.ParseDatetime(modEvt.CreatedAt) + if err != nil { + return false, err + } + reportDupePeriod := eng.Config.ReportDupePeriod + if reportDupePeriod == 0 { + reportDupePeriod = 1 * 24 * time.Hour + } + if time.Since(created.Time()) > reportDupePeriod { + continue + } + + // there is a recent report which is similar to this one + eng.Logger.Info("skipping duplicate account report due to API check") + return false, nil + } + + eng.Logger.Info("reporting account", "reasonType", mr.ReasonType, "comment", mr.Comment) + actionNewReportCount.WithLabelValues("account").Inc() + comment := "[automod] " + mr.Comment + _, err = toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventReport: &toolsozone.ModerationDefs_ModEventReport{ + Comment: &comment, + ReportType: &mr.ReasonType, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &comatproto.AdminDefs_RepoRef{ + Did: did.String(), + }, + }, + }) + if err != nil { + return false, err + } + return true, nil +} + +// Create a moderation report, but checks first if there was a similar recent one, and skips if so. +// +// Returns a bool indicating if a new report was created. +// +// TODO: merge this with createReportIfFresh() +func (eng *Engine) createRecordReportIfFresh(ctx context.Context, xrpcc *xrpc.Client, uri syntax.ATURI, cid *syntax.CID, mr ModReport) (bool, error) { + // before creating a report, query to see if automod has already reported this account in the past week for the same reason + // NOTE: this is running in an inner loop (if there are multiple reports), which is a bit inefficient, but seems acceptable + + resp, err := toolsozone.ModerationQueryEvents( + ctx, + xrpcc, + nil, // addedLabels []string + nil, // addedTags []string + "", // ageAssuranceState + "", // batchId string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + xrpcc.Auth.Did, // createdBy string + "", // cursor string + false, // hasComment bool + false, // includeAllUserRecords bool + 5, // limit int64 + nil, // modTool + nil, // policies []string + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + uri.String(), // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string + false, // withStrike bool + ) + if err != nil { + return false, err + } + for _, modEvt := range resp.Events { + // defensively ensure that our query params worked correctly + if modEvt.Event.ModerationDefs_ModEventReport == nil || modEvt.CreatedBy != xrpcc.Auth.Did || modEvt.Subject.RepoStrongRef == nil || modEvt.Subject.RepoStrongRef.Uri != uri.String() || (modEvt.Event.ModerationDefs_ModEventReport.ReportType != nil && *modEvt.Event.ModerationDefs_ModEventReport.ReportType != mr.ReasonType) { + continue + } + // igonre if older + created, err := syntax.ParseDatetime(modEvt.CreatedAt) + if err != nil { + return false, err + } + reportDupePeriod := eng.Config.ReportDupePeriod + if reportDupePeriod == 0 { + reportDupePeriod = 1 * 24 * time.Hour + } + if time.Since(created.Time()) > reportDupePeriod { + continue + } + + // there is a recent report which is similar to this one + eng.Logger.Info("skipping duplicate account report due to API check") + return false, nil + } + + eng.Logger.Info("reporting record", "reasonType", mr.ReasonType, "comment", mr.Comment) + actionNewReportCount.WithLabelValues("record").Inc() + comment := "[automod] " + mr.Comment + _, err = toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: xrpcc.Auth.Did, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventReport: &toolsozone.ModerationDefs_ModEventReport{ + Comment: &comment, + ReportType: &mr.ReasonType, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + RepoStrongRef: &comatproto.RepoStrongRef{ + Uri: uri.String(), + Cid: cid.String(), + }, + }, + }) + if err != nil { + return false, err + } + return true, nil +} diff --git a/automod/engine/report.go b/automod/engine/report.go new file mode 100644 index 000000000..52d59959f --- /dev/null +++ b/automod/engine/report.go @@ -0,0 +1,35 @@ +package engine + +// Simplified variant of input parameters for com.atproto.moderation.createReport, for internal tracking +type ModReport struct { + ReasonType string + Comment string +} + +var ( + ReportReasonSpam = "com.atproto.moderation.defs#reasonSpam" + ReportReasonViolation = "com.atproto.moderation.defs#reasonViolation" + ReportReasonMisleading = "com.atproto.moderation.defs#reasonMisleading" + ReportReasonSexual = "com.atproto.moderation.defs#reasonSexual" + ReportReasonRude = "com.atproto.moderation.defs#reasonRude" + ReportReasonOther = "com.atproto.moderation.defs#reasonOther" +) + +func ReasonShortName(reason string) string { + switch reason { + case ReportReasonSpam: + return "spam" + case ReportReasonViolation: + return "violation" + case ReportReasonMisleading: + return "misleading" + case ReportReasonSexual: + return "sexual" + case ReportReasonRude: + return "rude" + case ReportReasonOther: + return "other" + default: + return "unknown" + } +} diff --git a/automod/engine/ruleset.go b/automod/engine/ruleset.go new file mode 100644 index 000000000..1d06e23a5 --- /dev/null +++ b/automod/engine/ruleset.go @@ -0,0 +1,180 @@ +package engine + +import ( + "bytes" + "fmt" + "sync" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +// Holds configuration of which rules of various types should be run, and helps dispatch events to those rules. +type RuleSet struct { + PostRules []PostRuleFunc + ProfileRules []ProfileRuleFunc + RecordRules []RecordRuleFunc + RecordDeleteRules []RecordRuleFunc + IdentityRules []IdentityRuleFunc + AccountRules []AccountRuleFunc + BlobRules []BlobRuleFunc + OzoneEventRules []OzoneEventRuleFunc +} + +// Executes all the various record-related rules. Only dispatches execution, does no other de-dupe or pre/post processing. +func (r *RuleSet) CallRecordRules(c *RecordContext) error { + // first the generic rules + for _, f := range r.RecordRules { + err := f(c) + if err != nil { + c.Logger.Error("record rule execution failed", "err", err) + } + } + // then any record-type-specific rules + switch c.RecordOp.Collection.String() { + case "app.bsky.feed.post": + var post appbsky.FeedPost + if err := post.UnmarshalCBOR(bytes.NewReader(c.RecordOp.RecordCBOR)); err != nil { + return fmt.Errorf("failed to parse app.bsky.feed.post record: %v", err) + } + for _, f := range r.PostRules { + err := f(c, &post) + if err != nil { + c.Logger.Error("post rule execution failed", "err", err) + } + } + case "app.bsky.actor.profile": + var profile appbsky.ActorProfile + if err := profile.UnmarshalCBOR(bytes.NewReader(c.RecordOp.RecordCBOR)); err != nil { + return fmt.Errorf("failed to parse app.bsky.actor.profile record: %v", err) + } + for _, f := range r.ProfileRules { + err := f(c, &profile) + if err != nil { + c.Logger.Error("profile rule execution failed", "err", err) + } + } + } + // then blob rules, if any + if len(r.BlobRules) == 0 { + return nil + } + err := r.fetchAndProcessBlobs(c) + if err != nil { + c.Logger.Error("failed to fetch and process blobs", "err", err) + } + + return nil +} + +// NOTE: this will probably be removed and merged in to `CallRecordRules` +func (r *RuleSet) CallRecordDeleteRules(c *RecordContext) error { + for _, f := range r.RecordDeleteRules { + err := f(c) + if err != nil { + c.Logger.Error("record delete rule execution failed", "err", err) + } + } + return nil +} + +// Executes rules for identity update events. +func (r *RuleSet) CallIdentityRules(c *AccountContext) error { + for _, f := range r.IdentityRules { + err := f(c) + if err != nil { + c.Logger.Error("identity rule execution failed", "err", err) + } + } + return nil +} + +// Executes rules for account update events. +func (r *RuleSet) CallAccountRules(c *AccountContext) error { + for _, f := range r.AccountRules { + err := f(c) + if err != nil { + c.Logger.Error("account rule execution failed", "err", err) + } + } + return nil +} + +func (r *RuleSet) CallOzoneEventRules(c *OzoneEventContext) error { + for _, f := range r.OzoneEventRules { + err := f(c) + if err != nil { + c.Logger.Error("ozone event rule execution failed", "err", err) + } + } + return nil +} + +// high-level helper for fetching and processing blobs concurrently +func (r *RuleSet) fetchAndProcessBlobs(c *RecordContext) error { + + blobs, err := c.Blobs() + if err != nil { + return fmt.Errorf("failed to extract blobs from record: %w", err) + } + if len(blobs) == 0 { + return nil + } + + errChan := make(chan error, len(blobs)) + var wg sync.WaitGroup + for _, blob := range blobs { + wg.Add(1) + go func(blob lexutil.LexBlob) { + defer wg.Done() + data, err := c.fetchBlob(blob) + if err != nil { + errChan <- err + return + } + err = r.processBlob(c, blob, data) + if err != nil { + errChan <- err + return + } + }(blob) + + } + wg.Wait() + close(errChan) + + // check for errors + for err := range errChan { + if err != nil { + return err + } + } + return nil +} + +func (r *RuleSet) processBlob(c *RecordContext, blob lexutil.LexBlob, data []byte) error { + errChan := make(chan error, len(r.BlobRules)) + var wg sync.WaitGroup + for _, f := range r.BlobRules { + wg.Add(1) + go func(brf BlobRuleFunc) { + defer wg.Done() + err := brf(c, blob, data) + if err != nil { + errChan <- err + return + } + }(f) + } + + wg.Wait() + close(errChan) + + // check for errors + for err := range errChan { + if err != nil { + return err + } + } + return nil +} diff --git a/automod/engine/ruletypes.go b/automod/engine/ruletypes.go new file mode 100644 index 000000000..534f72833 --- /dev/null +++ b/automod/engine/ruletypes.go @@ -0,0 +1,14 @@ +package engine + +import ( + appbsky "github.com/bluesky-social/indigo/api/bsky" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +type IdentityRuleFunc = func(c *AccountContext) error +type AccountRuleFunc = func(c *AccountContext) error +type RecordRuleFunc = func(c *RecordContext) error +type PostRuleFunc = func(c *RecordContext, post *appbsky.FeedPost) error +type ProfileRuleFunc = func(c *RecordContext, profile *appbsky.ActorProfile) error +type BlobRuleFunc = func(c *RecordContext, blob lexutil.LexBlob, data []byte) error +type OzoneEventRuleFunc = func(c *OzoneEventContext) error diff --git a/automod/engine/slack.go b/automod/engine/slack.go new file mode 100644 index 000000000..902df5aeb --- /dev/null +++ b/automod/engine/slack.go @@ -0,0 +1,95 @@ +package engine + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" +) + +type SlackNotifier struct { + SlackWebhookURL string +} + +func (n *SlackNotifier) SendAccount(ctx context.Context, service string, c *AccountContext) error { + if service != "slack" { + return nil + } + msg := slackBody("⚠️ Automod Account Action ⚠️\n", c.Account, c.effects.AccountLabels, c.effects.RemovedAccountLabels, c.effects.AccountFlags, c.effects.AccountReports, c.effects.AccountTakedown) + c.Logger.Debug("sending slack notification") + return n.sendSlackMsg(ctx, msg) +} + +func (n *SlackNotifier) SendRecord(ctx context.Context, service string, c *RecordContext) error { + if service != "slack" { + return nil + } + atURI := fmt.Sprintf("at://%s/%s/%s", c.Account.Identity.DID, c.RecordOp.Collection, c.RecordOp.RecordKey) + msg := slackBody("⚠️ Automod Record Action ⚠️\n", c.Account, c.effects.RecordLabels, c.effects.RemovedRecordLabels, c.effects.RecordFlags, c.effects.RecordReports, c.effects.RecordTakedown) + msg += fmt.Sprintf("`%s`\n", atURI) + c.Logger.Debug("sending slack notification") + return n.sendSlackMsg(ctx, msg) +} + +type SlackWebhookBody struct { + Text string `json:"text"` +} + +// Sends a simple slack message to a channel via "incoming webhook". +// +// The slack incoming webhook must be already configured in the slack workplace. +func (n *SlackNotifier) sendSlackMsg(ctx context.Context, msg string) error { + // loosely based on: https://golangcode.com/send-slack-messages-without-a-library/ + + body, err := json.Marshal(SlackWebhookBody{Text: msg}) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, n.SlackWebhookURL, bytes.NewBuffer(body)) + if err != nil { + return err + } + req.Header.Add("Content-Type", "application/json") + client := http.DefaultClient + resp, err := client.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + if resp.StatusCode != 200 || buf.String() != "ok" { + return fmt.Errorf("failed slack webhook POST request. status=%d", resp.StatusCode) + } + return nil +} + +func slackBody(header string, acct AccountMeta, newLabels, rmdLabels, newFlags []string, newReports []ModReport, newTakedown bool) string { + msg := header + msg += fmt.Sprintf("`%s` / `%s` / / \n", + acct.Identity.DID, + acct.Identity.Handle, + acct.Identity.DID, + acct.Identity.DID, + ) + if len(newLabels) > 0 { + msg += fmt.Sprintf("Added Labels: `%s`\n", strings.Join(newLabels, ", ")) + } + if len(rmdLabels) > 0 { + msg += fmt.Sprintf("Removed Labels: `%s`\n", strings.Join(rmdLabels, ", ")) + } + if len(newFlags) > 0 { + msg += fmt.Sprintf("Flags: `%s`\n", strings.Join(newFlags, ", ")) + } + for _, rep := range newReports { + msg += fmt.Sprintf("Report `%s`: %s\n", rep.ReasonType, rep.Comment) + } + if newTakedown { + msg += "Takedown!\n" + } + return msg +} diff --git a/automod/engine/testing.go b/automod/engine/testing.go new file mode 100644 index 000000000..392977bab --- /dev/null +++ b/automod/engine/testing.go @@ -0,0 +1,76 @@ +package engine + +import ( + "log/slog" + "time" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod/cachestore" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/flagstore" + "github.com/bluesky-social/indigo/automod/setstore" +) + +var _ PostRuleFunc = simpleRule + +func simpleRule(c *RecordContext, post *appbsky.FeedPost) error { + for _, tag := range post.Tags { + if c.InSet("bad-hashtags", tag) { + c.AddRecordLabel("bad-hashtag") + break + } + } + for _, facet := range post.Facets { + for _, feat := range facet.Features { + if feat.RichtextFacet_Tag != nil { + tag := feat.RichtextFacet_Tag.Tag + if c.InSet("bad-hashtags", tag) { + c.AddRecordLabel("bad-hashtag") + break + } + } + } + } + return nil +} + +func EngineTestFixture() Engine { + rules := RuleSet{ + PostRules: []PostRuleFunc{ + simpleRule, + }, + } + cache := cachestore.NewMemCacheStore(10, time.Hour) + flags := flagstore.NewMemFlagStore() + sets := setstore.NewMemSetStore() + sets.Sets["bad-hashtags"] = make(map[string]bool) + sets.Sets["bad-hashtags"]["slur"] = true + sets.Sets["bad-words"] = make(map[string]bool) + sets.Sets["bad-words"]["hardr"] = true + sets.Sets["bad-words"]["hardestr"] = true + sets.Sets["worst-words"] = make(map[string]bool) + sets.Sets["worst-words"]["hardestr"] = true + dir := identity.NewMockDirectory() + id1 := identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + } + dir.Insert(id1) + eng := Engine{ + Logger: slog.Default(), + Directory: &dir, + Counters: countstore.NewMemCountStore(), + Sets: sets, + Flags: flags, + Cache: cache, + Rules: rules, + } + return eng +} + +// Helper to access the private effects field from a context. Intended for use in test code, *not* from rules. +func ExtractEffects(c *BaseContext) *Effects { + return c.effects +} diff --git a/automod/engine/util.go b/automod/engine/util.go new file mode 100644 index 000000000..e96c411d5 --- /dev/null +++ b/automod/engine/util.go @@ -0,0 +1,37 @@ +package engine + +import ( + "net/url" + "strings" +) + +func dedupeStrings(in []string) []string { + var out []string + seen := make(map[string]bool) + for _, v := range in { + if !seen[v] { + out = append(out, v) + seen[v] = true + } + } + return out +} + +// get the cid from a bluesky cdn url +func cidFromCdnUrl(str *string) *string { + if str == nil { + return nil + } + + u, err := url.Parse(*str) + if err != nil || u.Host != "cdn.bsky.app" { + return nil + } + + parts := strings.Split(u.Path, "/") + if len(parts) != 6 { + return nil + } + + return &strings.Split(parts[5], "@")[0] +} diff --git a/automod/flagstore/doc.go b/automod/flagstore/doc.go new file mode 100644 index 000000000..597d7450a --- /dev/null +++ b/automod/flagstore/doc.go @@ -0,0 +1,2 @@ +// Interface for storing "flags", a form of private automod metadata. +package flagstore diff --git a/automod/flagstore/flagstore.go b/automod/flagstore/flagstore.go new file mode 100644 index 000000000..25212643c --- /dev/null +++ b/automod/flagstore/flagstore.go @@ -0,0 +1,11 @@ +package flagstore + +import ( + "context" +) + +type FlagStore interface { + Get(ctx context.Context, key string) ([]string, error) + Add(ctx context.Context, key string, flags []string) error + Remove(ctx context.Context, key string, flags []string) error +} diff --git a/automod/flagstore/flagstore_mem.go b/automod/flagstore/flagstore_mem.go new file mode 100644 index 000000000..bc068475c --- /dev/null +++ b/automod/flagstore/flagstore_mem.go @@ -0,0 +1,58 @@ +package flagstore + +import ( + "context" +) + +type MemFlagStore struct { + Data map[string][]string +} + +func NewMemFlagStore() MemFlagStore { + return MemFlagStore{ + Data: make(map[string][]string), + } +} + +func (s MemFlagStore) Get(ctx context.Context, key string) ([]string, error) { + v, ok := s.Data[key] + if !ok { + return []string{}, nil + } + return v, nil +} + +func (s MemFlagStore) Add(ctx context.Context, key string, flags []string) error { + v, ok := s.Data[key] + if !ok { + v = []string{} + } + v = append(v, flags...) + v = dedupeStrings(v) + s.Data[key] = v + return nil +} + +// does not error if flags not in set +func (s MemFlagStore) Remove(ctx context.Context, key string, flags []string) error { + if len(flags) == 0 { + return nil + } + v, ok := s.Data[key] + if !ok { + v = []string{} + } + m := make(map[string]bool, len(v)) + for _, f := range v { + m[f] = true + } + for _, f := range flags { + delete(m, f) + } + out := []string{} + for f, _ := range m { + out = append(out, f) + } + s.Data[key] = out + return nil +} diff --git a/automod/flagstore/flagstore_redis.go b/automod/flagstore/flagstore_redis.go new file mode 100644 index 000000000..ac1ee1117 --- /dev/null +++ b/automod/flagstore/flagstore_redis.go @@ -0,0 +1,66 @@ +package flagstore + +import ( + "context" + + "github.com/redis/go-redis/v9" +) + +var redisFlagsPrefix string = "flags/" + +type RedisFlagStore struct { + Client *redis.Client +} + +func NewRedisFlagStore(redisURL string) (*RedisFlagStore, error) { + ctx := context.Background() + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, err + } + rdb := redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(ctx).Result() + if err != nil { + return nil, err + } + rcs := RedisFlagStore{ + Client: rdb, + } + return &rcs, nil +} + +func (s *RedisFlagStore) Get(ctx context.Context, key string) ([]string, error) { + rkey := redisFlagsPrefix + key + l, err := s.Client.SMembers(ctx, rkey).Result() + if err == redis.Nil { + return []string{}, nil + } else if err != nil { + return nil, err + } + return l, nil +} + +func (s *RedisFlagStore) Add(ctx context.Context, key string, flags []string) error { + if len(flags) == 0 { + return nil + } + l := []interface{}{} + for _, v := range flags { + l = append(l, v) + } + rkey := redisFlagsPrefix + key + return s.Client.SAdd(ctx, rkey, l...).Err() +} + +func (s *RedisFlagStore) Remove(ctx context.Context, key string, flags []string) error { + if len(flags) == 0 { + return nil + } + l := []interface{}{} + for _, v := range flags { + l = append(l, v) + } + rkey := redisFlagsPrefix + key + return s.Client.SRem(ctx, rkey, l...).Err() +} diff --git a/automod/flagstore/flagstore_redis_test.go b/automod/flagstore/flagstore_redis_test.go new file mode 100644 index 000000000..fee97d6f1 --- /dev/null +++ b/automod/flagstore/flagstore_redis_test.go @@ -0,0 +1,35 @@ +package flagstore + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRedisFlagStoreBasics(t *testing.T) { + t.Skip("live test, need redis running locally") + assert := assert.New(t) + ctx := context.Background() + + fs, err := NewRedisFlagStore("redis://localhost:6379/0") + if err != nil { + t.Fail() + } + + l, err := fs.Get(ctx, "test1") + assert.NoError(err) + assert.Empty(l) + + assert.NoError(fs.Add(ctx, "test1", []string{"red", "green"})) + assert.NoError(fs.Add(ctx, "test1", []string{"red", "blue"})) + l, err = fs.Get(ctx, "test1") + assert.NoError(err) + assert.Equal(3, len(l)) + + assert.NoError(fs.Remove(ctx, "test1", []string{"red", "blue", "orange"})) + l, err = fs.Get(ctx, "test1") + assert.NoError(err) + assert.Equal([]string{"green"}, l) + assert.NoError(fs.Remove(ctx, "test1", []string{"green"})) +} diff --git a/automod/flagstore/flagstore_test.go b/automod/flagstore/flagstore_test.go new file mode 100644 index 000000000..c3bade80c --- /dev/null +++ b/automod/flagstore/flagstore_test.go @@ -0,0 +1,30 @@ +package flagstore + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFlagStoreBasics(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + fs := NewMemFlagStore() + + l, err := fs.Get(ctx, "test1") + assert.NoError(err) + assert.Empty(l) + + assert.NoError(fs.Add(ctx, "test1", []string{"red", "green"})) + assert.NoError(fs.Add(ctx, "test1", []string{"red", "blue"})) + l, err = fs.Get(ctx, "test1") + assert.NoError(err) + assert.Equal(3, len(l)) + + assert.NoError(fs.Remove(ctx, "test1", []string{"red", "blue"})) + l, err = fs.Get(ctx, "test1") + assert.NoError(err) + assert.Equal([]string{"green"}, l) +} diff --git a/automod/flagstore/util.go b/automod/flagstore/util.go new file mode 100644 index 000000000..923a55c6d --- /dev/null +++ b/automod/flagstore/util.go @@ -0,0 +1,13 @@ +package flagstore + +func dedupeStrings(in []string) []string { + var out []string + seen := make(map[string]bool) + for _, v := range in { + if !seen[v] { + out = append(out, v) + seen[v] = true + } + } + return out +} diff --git a/automod/helpers/account.go b/automod/helpers/account.go new file mode 100644 index 000000000..2a1a275cb --- /dev/null +++ b/automod/helpers/account.go @@ -0,0 +1,49 @@ +package helpers + +import ( + "time" + + "github.com/bluesky-social/indigo/automod" +) + +// no accounts exist before this time +var atprotoAccountEpoch = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + +// returns true if account creation timestamp is plausible: not-nil, not in distant past, not in the future +func plausibleAccountCreation(when *time.Time) bool { + if when == nil { + return false + } + // this is mostly to check for misconfigurations or null values (eg, UNIX epoch zero means "unknown" not actually 1970) + if !when.After(atprotoAccountEpoch) { + return false + } + // a timestamp in the future would also indicate some misconfiguration + if when.After(time.Now().Add(time.Hour)) { + return false + } + return true +} + +// checks if account was created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false' +func AccountIsYoungerThan(c *automod.AccountContext, age time.Duration) bool { + // TODO: consider swapping priority order here (and below) + if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) { + return time.Since(*c.Account.CreatedAt) < age + } + if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) { + return time.Since(*c.Account.Private.IndexedAt) < age + } + return false +} + +// checks if account was *not* created recently, based on either public or private account metadata. if metadata isn't available at all, or seems bogus, returns 'false' +func AccountIsOlderThan(c *automod.AccountContext, age time.Duration) bool { + if c.Account.CreatedAt != nil && plausibleAccountCreation(c.Account.CreatedAt) { + return time.Since(*c.Account.CreatedAt) >= age + } + if c.Account.Private != nil && plausibleAccountCreation(c.Account.Private.IndexedAt) { + return time.Since(*c.Account.Private.IndexedAt) >= age + } + return false +} diff --git a/automod/helpers/account_test.go b/automod/helpers/account_test.go new file mode 100644 index 000000000..c949eb77c --- /dev/null +++ b/automod/helpers/account_test.go @@ -0,0 +1,61 @@ +package helpers + +import ( + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/stretchr/testify/assert" +) + +func TestAccountIsYoungerThan(t *testing.T) { + assert := assert.New(t) + + am := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + Profile: automod.ProfileSummary{}, + Private: nil, + } + now := time.Now() + ac := automod.AccountContext{ + Account: am, + } + assert.False(AccountIsYoungerThan(&ac, time.Hour)) + assert.False(AccountIsOlderThan(&ac, time.Hour)) + + ac.Account.CreatedAt = &now + assert.True(AccountIsYoungerThan(&ac, time.Hour)) + assert.False(AccountIsOlderThan(&ac, time.Hour)) + + yesterday := time.Now().Add(-1 * time.Hour * 24) + ac.Account.CreatedAt = &yesterday + assert.False(AccountIsYoungerThan(&ac, time.Hour)) + assert.True(AccountIsOlderThan(&ac, time.Hour)) + + old := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC) + ac.Account.CreatedAt = &old + assert.False(AccountIsYoungerThan(&ac, time.Hour)) + assert.False(AccountIsYoungerThan(&ac, time.Hour*24*365*100)) + assert.False(AccountIsOlderThan(&ac, time.Hour)) + assert.False(AccountIsOlderThan(&ac, time.Hour*24*365*100)) + + future := time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC) + ac.Account.CreatedAt = &future + assert.False(AccountIsYoungerThan(&ac, time.Hour)) + assert.False(AccountIsOlderThan(&ac, time.Hour)) + + ac.Account.CreatedAt = nil + ac.Account.Private = &automod.AccountPrivate{ + Email: "account@example.com", + IndexedAt: &yesterday, + } + assert.True(AccountIsYoungerThan(&ac, 48*time.Hour)) + assert.False(AccountIsYoungerThan(&ac, time.Hour)) + assert.True(AccountIsOlderThan(&ac, time.Hour)) + assert.False(AccountIsOlderThan(&ac, 48*time.Hour)) +} diff --git a/automod/helpers/bsky.go b/automod/helpers/bsky.go new file mode 100644 index 000000000..a38ecda92 --- /dev/null +++ b/automod/helpers/bsky.go @@ -0,0 +1,267 @@ +package helpers + +import ( + "fmt" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/keyword" +) + +func ExtractHashtagsPost(post *appbsky.FeedPost) []string { + var tags []string + tags = append(tags, post.Tags...) + for _, facet := range post.Facets { + for _, feat := range facet.Features { + if feat.RichtextFacet_Tag != nil { + tags = append(tags, feat.RichtextFacet_Tag.Tag) + } + } + } + return DedupeStrings(tags) +} + +func NormalizeHashtag(raw string) string { + return keyword.Slugify(raw) +} + +type PostFacet struct { + Text string + URL *string + DID *string + Tag *string +} + +func ExtractFacets(post *appbsky.FeedPost) ([]PostFacet, error) { + var out []PostFacet + + for _, facet := range post.Facets { + for _, feat := range facet.Features { + if int(facet.Index.ByteEnd) > len([]byte(post.Text)) || facet.Index.ByteStart > facet.Index.ByteEnd { + return nil, fmt.Errorf("invalid facet byte range") + } + + txt := string([]byte(post.Text)[facet.Index.ByteStart:facet.Index.ByteEnd]) + if txt == "" { + return nil, fmt.Errorf("empty facet text") + } + + if feat.RichtextFacet_Link != nil { + out = append(out, PostFacet{ + Text: txt, + URL: &feat.RichtextFacet_Link.Uri, + }) + } + if feat.RichtextFacet_Tag != nil { + out = append(out, PostFacet{ + Text: txt, + Tag: &feat.RichtextFacet_Tag.Tag, + }) + } + if feat.RichtextFacet_Mention != nil { + out = append(out, PostFacet{ + Text: txt, + DID: &feat.RichtextFacet_Mention.Did, + }) + } + } + } + return out, nil +} + +func ExtractPostBlobCIDsPost(post *appbsky.FeedPost) []string { + var out []string + if post.Embed.EmbedImages != nil { + for _, img := range post.Embed.EmbedImages.Images { + out = append(out, img.Image.Ref.String()) + } + } + if post.Embed.EmbedRecordWithMedia != nil { + media := post.Embed.EmbedRecordWithMedia.Media + if media.EmbedImages != nil { + for _, img := range media.EmbedImages.Images { + out = append(out, img.Image.Ref.String()) + } + } + } + return DedupeStrings(out) +} + +func ExtractBlobCIDsProfile(profile *appbsky.ActorProfile) []string { + var out []string + if profile.Avatar != nil { + out = append(out, profile.Avatar.Ref.String()) + } + if profile.Banner != nil { + out = append(out, profile.Banner.Ref.String()) + } + return DedupeStrings(out) +} + +func ExtractTextTokensPost(post *appbsky.FeedPost) []string { + s := post.Text + if post.Embed != nil { + if post.Embed.EmbedImages != nil { + for _, img := range post.Embed.EmbedImages.Images { + if img.Alt != "" { + s += " " + img.Alt + } + } + } + if post.Embed.EmbedRecordWithMedia != nil { + media := post.Embed.EmbedRecordWithMedia.Media + if media.EmbedImages != nil { + for _, img := range media.EmbedImages.Images { + if img.Alt != "" { + s += " " + img.Alt + } + } + } + } + } + return keyword.TokenizeText(s) +} + +func ExtractTextTokensProfile(profile *appbsky.ActorProfile) []string { + s := "" + if profile.Description != nil { + s += " " + *profile.Description + } + if profile.DisplayName != nil { + s += " " + *profile.DisplayName + } + return keyword.TokenizeText(s) +} + +func ExtractTextURLsProfile(profile *appbsky.ActorProfile) []string { + s := "" + if profile.Description != nil { + s += " " + *profile.Description + } + if profile.DisplayName != nil { + s += " " + *profile.DisplayName + } + return ExtractTextURLs(s) +} + +// checks if the post event is a reply post for which the author is replying to themselves, or author is the root author (OP) +func IsSelfThread(c *automod.RecordContext, post *appbsky.FeedPost) bool { + if post.Reply == nil { + return false + } + did := c.Account.Identity.DID.String() + parentURI, err := syntax.ParseATURI(post.Reply.Parent.Uri) + if err != nil { + return false + } + rootURI, err := syntax.ParseATURI(post.Reply.Root.Uri) + if err != nil { + return false + } + + if parentURI.Authority().String() == did || rootURI.Authority().String() == did { + return true + } + return false +} + +func ParentOrRootIsFollower(c *automod.RecordContext, post *appbsky.FeedPost) bool { + if post.Reply == nil || IsSelfThread(c, post) { + return false + } + + parentURI, err := syntax.ParseATURI(post.Reply.Parent.Uri) + if err != nil { + c.Logger.Warn("failed to parse reply AT-URI", "uri", post.Reply.Parent.Uri) + return false + } + parentDID, err := parentURI.Authority().AsDID() + if err != nil { + c.Logger.Warn("reply AT-URI authority not a DID", "uri", post.Reply.Parent.Uri) + return false + } + + rel := c.GetAccountRelationship(parentDID) + if rel.FollowedBy { + return true + } + + rootURI, err := syntax.ParseATURI(post.Reply.Root.Uri) + if err != nil { + c.Logger.Warn("failed to parse reply AT-URI", "uri", post.Reply.Root.Uri) + return false + } + rootDID, err := rootURI.Authority().AsDID() + if err != nil { + c.Logger.Warn("reply AT-URI authority not a DID", "uri", post.Reply.Root.Uri) + return false + } + + if rootDID == parentDID { + return false + } + + rel = c.GetAccountRelationship(rootDID) + if rel.FollowedBy { + return true + } + return false +} + +func PostParentOrRootIsDid(post *appbsky.FeedPost, did string) bool { + if post.Reply == nil { + return false + } + + rootUri, err := syntax.ParseATURI(post.Reply.Root.Uri) + if err != nil || !rootUri.Authority().IsDID() { + return false + } + + parentUri, err := syntax.ParseATURI(post.Reply.Parent.Uri) + if err != nil || !parentUri.Authority().IsDID() { + return false + } + + return rootUri.Authority().String() == did || parentUri.Authority().String() == did +} + +func PostParentOrRootIsAnyDid(post *appbsky.FeedPost, dids []string) bool { + if post.Reply == nil { + return false + } + + for _, did := range dids { + if PostParentOrRootIsDid(post, did) { + return true + } + } + + return false +} + +func PostMentionsDid(post *appbsky.FeedPost, did string) bool { + facets, err := ExtractFacets(post) + if err != nil { + return false + } + + for _, facet := range facets { + if facet.DID != nil && *facet.DID == did { + return true + } + } + + return false +} + +func PostMentionsAnyDid(post *appbsky.FeedPost, dids []string) bool { + for _, did := range dids { + if PostMentionsDid(post, did) { + return true + } + } + + return false +} diff --git a/automod/helpers/bsky_test.go b/automod/helpers/bsky_test.go new file mode 100644 index 000000000..b5d6cb242 --- /dev/null +++ b/automod/helpers/bsky_test.go @@ -0,0 +1,141 @@ +package helpers + +import ( + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParentOrRootIsDid(t *testing.T) { + assert := assert.New(t) + + post1 := &appbsky.FeedPost{ + Text: "some random post that i dreamt up last night, idk", + Reply: &appbsky.FeedPost_ReplyRef{ + Root: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:abc123/app.bsky.feed.post/rkey123", + }, + Parent: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:abc123/app.bsky.feed.post/rkey123", + }, + }, + } + + post2 := &appbsky.FeedPost{ + Text: "some random post that i dreamt up last night, idk", + Reply: &appbsky.FeedPost_ReplyRef{ + Root: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:321abc/app.bsky.feed.post/rkey123", + }, + Parent: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:abc123/app.bsky.feed.post/rkey123", + }, + }, + } + + post3 := &appbsky.FeedPost{ + Text: "some random post that i dreamt up last night, idk", + Reply: &appbsky.FeedPost_ReplyRef{ + Root: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:abc123/app.bsky.feed.post/rkey123", + }, + Parent: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:321abc/app.bsky.feed.post/rkey123", + }, + }, + } + + post4 := &appbsky.FeedPost{ + Text: "some random post that i dreamt up last night, idk", + Reply: &appbsky.FeedPost_ReplyRef{ + Root: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:321abc/app.bsky.feed.post/rkey123", + }, + Parent: &comatproto.RepoStrongRef{ + Uri: "at://did:plc:321abc/app.bsky.feed.post/rkey123", + }, + }, + } + + assert.True(PostParentOrRootIsDid(post1, "did:plc:abc123")) + assert.False(PostParentOrRootIsDid(post1, "did:plc:321abc")) + + assert.True(PostParentOrRootIsDid(post2, "did:plc:abc123")) + assert.True(PostParentOrRootIsDid(post2, "did:plc:321abc")) + + assert.True(PostParentOrRootIsDid(post3, "did:plc:abc123")) + assert.True(PostParentOrRootIsDid(post3, "did:plc:321abc")) + + assert.False(PostParentOrRootIsDid(post4, "did:plc:abc123")) + assert.True(PostParentOrRootIsDid(post4, "did:plc:321abc")) + + didList1 := []string{ + "did:plc:cba321", + "did:web:bsky.app", + "did:plc:abc123", + } + + didList2 := []string{ + "did:plc:321cba", + "did:web:bsky.app", + "did:plc:123abc", + } + + assert.True(PostParentOrRootIsAnyDid(post1, didList1)) + assert.False(PostParentOrRootIsAnyDid(post1, didList2)) +} + +func TestPostMentionsDid(t *testing.T) { + assert := assert.New(t) + + post := &appbsky.FeedPost{ + Text: "@hailey.at what is upppp also hello to @darthbluesky.bsky.social", + Facets: []*appbsky.RichtextFacet{ + { + Features: []*appbsky.RichtextFacet_Features_Elem{ + { + RichtextFacet_Mention: &appbsky.RichtextFacet_Mention{ + Did: "did:plc:abc123", + }, + }, + }, + Index: &appbsky.RichtextFacet_ByteSlice{ + ByteStart: 0, + ByteEnd: 9, + }, + }, + { + Features: []*appbsky.RichtextFacet_Features_Elem{ + { + RichtextFacet_Mention: &appbsky.RichtextFacet_Mention{ + Did: "did:plc:abc456", + }, + }, + }, + Index: &appbsky.RichtextFacet_ByteSlice{ + ByteStart: 39, + ByteEnd: 63, + }, + }, + }, + } + assert.True(PostMentionsDid(post, "did:plc:abc123")) + assert.False(PostMentionsDid(post, "did:plc:cba321")) + + didList1 := []string{ + "did:plc:cba321", + "did:web:bsky.app", + "did:plc:abc456", + } + + didList2 := []string{ + "did:plc:321cba", + "did:web:bsky.app", + "did:plc:123abc", + } + + assert.True(PostMentionsAnyDid(post, didList1)) + assert.False(PostMentionsAnyDid(post, didList2)) +} diff --git a/automod/helpers/text.go b/automod/helpers/text.go new file mode 100644 index 000000000..412eb9c8c --- /dev/null +++ b/automod/helpers/text.go @@ -0,0 +1,35 @@ +package helpers + +import ( + "fmt" + "regexp" + + "github.com/spaolacci/murmur3" +) + +func DedupeStrings(in []string) []string { + var out []string + seen := make(map[string]bool) + for _, v := range in { + if !seen[v] { + out = append(out, v) + seen[v] = true + } + } + return out +} + +// returns a fast, compact hash of a string +// +// current implementation uses murmur3, default seed, and hex encoding +func HashOfString(s string) string { + val := murmur3.Sum64([]byte(s)) + return fmt.Sprintf("%016x", val) +} + +// based on: https://stackoverflow.com/a/48769624, with no trailing period allowed +var urlRegex = regexp.MustCompile(`(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-&?=%.]*[\w/\-&?=%]+`) + +func ExtractTextURLs(raw string) []string { + return urlRegex.FindAllString(raw, -1) +} diff --git a/automod/helpers/text_test.go b/automod/helpers/text_test.go new file mode 100644 index 000000000..ef219155e --- /dev/null +++ b/automod/helpers/text_test.go @@ -0,0 +1,64 @@ +package helpers + +import ( + "testing" + + "github.com/bluesky-social/indigo/automod/keyword" + + "github.com/stretchr/testify/assert" +) + +func TestTokenizeText(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + s string + out []string + }{ + { + s: "1 'Two' three!", + out: []string{"1", "two", "three"}, + }, + { + s: " foo1;bar2,baz3...", + out: []string{"foo1", "bar2", "baz3"}, + }, + { + s: "https://example.com/index.html", + out: []string{"https", "example", "com", "index", "html"}, + }, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, keyword.TokenizeText(fix.s)) + } +} + +func TestExtractURL(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + s string + out []string + }{ + { + s: "this is a description with example.com mentioned in the middle", + out: []string{"example.com"}, + }, + { + s: "this is another example with https://en.wikipedia.org/index.html: and archive.org, and https://eff.org/... and bsky.app.", + out: []string{"https://en.wikipedia.org/index.html", "archive.org", "https://eff.org/", "bsky.app"}, + }, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, ExtractTextURLs(fix.s)) + } +} + +func TestHashOfString(t *testing.T) { + assert := assert.New(t) + + // hashing function should be consistent over time + assert.Equal("4e6f69c0e3d10992", HashOfString("dummy-value")) +} diff --git a/automod/keyword/cmd/kw-cli/main.go b/automod/keyword/cmd/kw-cli/main.go new file mode 100644 index 000000000..4961737bf --- /dev/null +++ b/automod/keyword/cmd/kw-cli/main.go @@ -0,0 +1,96 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "log/slog" + "os" + + "github.com/bluesky-social/indigo/automod/keyword" + "github.com/bluesky-social/indigo/automod/setstore" + + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "kw-cli", + Usage: "informal debugging CLI tool for keyword matching", + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "fuzzy", + Usage: "reads lines of text from stdin, runs regex fuzzy matching, outputs matches", + Action: runFuzzy, + }, + &cli.Command{ + Name: "tokens", + Usage: "reads lines of text from stdin, tokenizes and matches against set", + Action: runTokens, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "json-set-file", + Usage: "path to JSON file containing bad word sets", + Value: "automod/rules/example_sets.json", + }, + &cli.StringFlag{ + Name: "set-name", + Usage: "which set within the set file to use", + Value: "bad-words", + }, + &cli.BoolFlag{ + Name: "identifiers", + Usage: "whether to parse the line as identifiers (instead of text)", + }, + }, + }, + } + h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + slog.SetDefault(slog.New(h)) + if err := app.Run(context.Background(), os.Args); err != nil { + slog.Error("command failed", "error", err) + os.Exit(-1) + } +} + +func runFuzzy(ctx context.Context, cmd *cli.Command) error { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + line := scanner.Text() + word := keyword.SlugContainsExplicitSlur(keyword.Slugify(line)) + if word != "" { + fmt.Printf("MATCH\t%s\t%s\n", word, line) + } + } + return nil +} + +func runTokens(ctx context.Context, cmd *cli.Command) error { + sets := setstore.NewMemSetStore() + if err := sets.LoadFromFileJSON(cmd.String("json-set-file")); err != nil { + return err + } + setName := cmd.String("set-name") + identMode := cmd.Bool("identifiers") + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + line := scanner.Text() + var tokens []string + if identMode { + tokens = keyword.TokenizeIdentifier(line) + } else { + tokens = keyword.TokenizeText(line) + } + for _, tok := range tokens { + match, err := sets.InSet(ctx, setName, tok) + if err != nil { + return err + } + if match { + fmt.Printf("MATCH\t%s\t%s\n", tok, line) + } + } + } + return nil +} diff --git a/automod/keyword/doc.go b/automod/keyword/doc.go new file mode 100644 index 000000000..1445adaf6 --- /dev/null +++ b/automod/keyword/doc.go @@ -0,0 +1,2 @@ +// String processing helpers for doing fuzzy detection and normalized token matching against keyword lists. +package keyword diff --git a/automod/keyword/keyword.go b/automod/keyword/keyword.go new file mode 100644 index 000000000..dde62a69b --- /dev/null +++ b/automod/keyword/keyword.go @@ -0,0 +1,11 @@ +package keyword + +// Helper to check a single token against a list of tokens +func TokenInSet(tok string, set []string) bool { + for _, v := range set { + if tok == v { + return true + } + } + return false +} diff --git a/automod/keyword/keyword_test.go b/automod/keyword/keyword_test.go new file mode 100644 index 000000000..66880f257 --- /dev/null +++ b/automod/keyword/keyword_test.go @@ -0,0 +1,20 @@ +package keyword + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTokenInSet(t *testing.T) { + assert := assert.New(t) + + keywords := []string{ + "example", + "bunch", + } + + assert.True(TokenInSet("example", keywords)) + assert.False(TokenInSet("Example", keywords)) + assert.False(TokenInSet("elephant", keywords)) +} diff --git a/automod/keyword/slugify.go b/automod/keyword/slugify.go new file mode 100644 index 000000000..45be8e326 --- /dev/null +++ b/automod/keyword/slugify.go @@ -0,0 +1,13 @@ +package keyword + +import ( + "regexp" + "strings" +) + +var nonSlugChars = regexp.MustCompile(`[^\pL\pN]+`) + +// Takes an arbitrary string (eg, an identifier or free-form text) and returns a version with all non-letter, non-digit characters removed, and all lower-case +func Slugify(orig string) string { + return strings.ToLower(nonSlugChars.ReplaceAllString(orig, "")) +} diff --git a/automod/keyword/slur_regex.go b/automod/keyword/slur_regex.go new file mode 100644 index 000000000..ed27540e2 --- /dev/null +++ b/automod/keyword/slur_regex.go @@ -0,0 +1,42 @@ +package keyword + +import ( + "regexp" +) + +// regexes taken from: https://github.com/Blank-Cheque/Slurs +var explicitSlurRegexes = map[string]*regexp.Regexp{ + "chink": regexp.MustCompile("[cĆćĈĉČčĊċÇçḈḉȻȼꞒꞓꟄꞔƇƈɕ][hĤĥȞȟḦḧḢḣḨḩḤḥḪḫH̱ẖĦħⱧⱨꞪɦꞕΗНн][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıＩｉ1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴＬｌ][nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпＮｎ][kḰḱǨǩĶķḲḳḴḵƘƙⱩⱪᶄꝀꝁꝂꝃꝄꝅꞢꞣ][sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?"), + // modified to not match "cocoon", "raccoon", "racoon", or "tycoon" + "coon": regexp.MustCompile("(^|[^cayo])[cĆćĈĉČčĊċÇçḈḉȻȼꞒꞓꟄꞔƇƈɕ][ÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺＯｏ0]{2}[nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпＮｎ][sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?"), + "faggot": regexp.MustCompile("[fḞḟƑƒꞘꞙᵮᶂ][aÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚＡａ@4][gǴǵĞğĜĝǦǧĠġG̃g̃ĢģḠḡǤǥꞠꞡƓɠᶃꬶＧｇqꝖꝗꝘꝙɋʠ]{2}([ÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺＯｏ0e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳＥｅiÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıＩｉ1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴＬｌ][tŤťṪṫŢţṬṭȚțṰṱṮṯŦŧȾⱦ ƬƭƮʈT̈ẗᵵƫȶ]{1,2}([rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][yÝýỲỳŶŷY̊ẙŸÿỸỹẎẏȲȳỶỷỴỵɎɏƳƴỾỿ]|[rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıＩｉ1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴＬｌ][e3ЄєЕеÉéÈèĔĕÊêẾế ỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳＥｅ])?)?[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?"), + "kike": regexp.MustCompile("[kḰḱǨǩĶķḲḳḴḵƘƙⱩⱪᶄꝀꝁꝂꝃꝄꝅꞢꞣ][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıＩｉ1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴＬｌyÝýỲỳŶŷY̊ẙŸÿỸỹẎẏȲȳỶỷỴỵɎɏƳƴỾỿ][kḰḱǨǩĶķḲḳḴḵƘƙⱩⱪᶄꝀꝁꝂꝃꝄꝅꞢꞣ][e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝ ĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳＥｅ]([rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][yÝýỲỳŶŷY̊ẙŸÿỸỹẎẏȲȳỶỷỴỵɎɏƳƴỾỿ]|[rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıＩｉ1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫ ɬꞎꬷꬸꬹᶅɭȴＬｌ][e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳＥｅ])?[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]*"), + // modified to not match "snigger" + "nigger": regexp.MustCompile("(^|[^s])[nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпＮｎ][iÍíi̇́Ììi̇̀ĬĭÎîǏǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıＩｉ1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴＬｌoÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺＯｏІіa4ÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚＡａ][gǴǵĞğĜĝǦǧĠġG̃g̃ĢģḠḡǤǥꞠꞡƓɠᶃꬶＧｇqꝖꝗꝘꝙɋʠ]{2}([e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳＥｅaÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚＡａ][rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ ]?|n[ÓóÒòŎŏÔôỐốỒồỖỗỔổǑǒÖöȪȫŐőÕõṌṍṎṏȬȭȮȯO͘o͘ȰȱØøǾǿǪǫǬǭŌōṒṓṐṑỎỏȌȍȎȏƠơỚớỜờỠỡỞởỢợỌọỘộO̩o̩Ò̩ò̩Ó̩ó̩ƟɵꝊꝋꝌꝍⱺＯｏ0][gǴǵĞğĜĝǦǧĠġG̃g̃ĢģḠḡǤǥꞠꞡƓɠᶃꬶＧｇqꝖꝗꝘꝙɋʠ]|[a4ÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚ Ａａ])[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?"), + "tranny": regexp.MustCompile("[tŤťṪṫŢţṬṭȚțṰṱṮṯŦŧȾⱦƬƭƮʈT̈ẗᵵƫȶ][rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ][aÁáÀàĂăẮắẰằẴẵẲẳÂâẤấẦầẪẫẨẩǍǎÅåǺǻÄäǞǟÃãȦȧǠǡĄąĄ́ą́Ą̃ą̃ĀāĀ̀ā̀ẢảȀȁA̋a̋ȂȃẠạẶặẬậḀḁȺⱥꞺꞻᶏẚＡａ4]+[nŃńǸǹŇňÑñṄṅŅņṆṇṊṋṈṉN̈n̈ƝɲŊŋꞐꞑꞤꞥᵰᶇɳȵꬻꬼИиПпＮｎ]{1,2}([iÍíi̇́Ììi̇̀ĬĭÎîǏ ǐÏïḮḯĨĩi̇̃ĮįĮ́į̇́Į̃į̇̃ĪīĪ̀ī̀ỈỉȈȉI̋i̋ȊȋỊịꞼꞽḬḭƗɨᶖİiIıＩｉ1lĺľļḷḹl̃ḽḻłŀƚꝉⱡɫɬꞎꬷꬸꬹᶅɭȴＬｌ][e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳＥｅ]|[yÝýỲỳŶŷY̊ẙŸÿỸỹẎẏȲȳỶỷỴỵɎɏƳƴỾỿ]|[e3ЄєЕеÉéÈèĔĕÊêẾếỀềỄễ ỂểÊ̄ê̄Ê̌ê̌ĚěËëẼẽĖėĖ́ė́Ė̃ė̃ȨȩḜḝĘęĘ́ę́Ę̃ę̃ĒēḖḗḔḕẺẻȄȅE̋e̋ȆȇẸẹỆệḘḙḚḛɆɇE̩e̩È̩è̩É̩é̩ᶒⱸꬴꬳＥｅ][rŔŕŘřṘṙŖŗȐȑȒȓṚṛṜṝṞṟR̃r̃ɌɍꞦꞧⱤɽᵲᶉꭉ])[sŚśṤṥŜŝŠšṦṧṠṡŞşṢṣṨṩȘșS̩s̩ꞨꞩⱾȿꟅʂᶊᵴ]?"), +} + +// For a small set of frequently-abused explicit slurs, checks for a of permissive set of "l33t-speak" variations of the keyword. This is intended to be used with pre-processed "slugs", which are strings with all whitespace, punctuation, and other characters removed. These could be pre-processed identifiers (like handles or record keys), or pre-processed free-form text. +// +// If there is a match, returns a plan-text version of the slur. +// +// This is a loose port of the 'hasExplicitSlur' function from the `@atproto/pds` TypeScript package. +func SlugContainsExplicitSlur(raw string) string { + for word, r := range explicitSlurRegexes { + if r.MatchString(raw) { + return word + } + } + return "" +} + +// Variant of `SlugContainsExplicitSlur` where the entire slug must match. +func SlugIsExplicitSlur(raw string) string { + for word, r := range explicitSlurRegexes { + m := r.FindString(raw) + if m != "" && m == raw { + return word + } + } + return "" +} diff --git a/automod/keyword/slur_regex_test.go b/automod/keyword/slur_regex_test.go new file mode 100644 index 000000000..ce953948d --- /dev/null +++ b/automod/keyword/slur_regex_test.go @@ -0,0 +1,70 @@ +package keyword + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSlugContainsExplicitSlur(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + text string + contains string + is string + }{ + {contains: "", is: "", text: ""}, + {contains: "", is: "", text: "hello"}, + {contains: "chink", is: "chink", text: "chink"}, + {contains: "faggot", is: "faggot", text: "faggot"}, + {contains: "faggot", is: "faggot", text: "f4gg0t"}, + {contains: "coon", is: "coon", text: "coon"}, + {contains: "coon", is: "coon", text: "coons"}, + {contains: "", is: "", text: "raccoon"}, + {contains: "", is: "", text: "racoon"}, + {contains: "", is: "", text: "tycoon"}, + {contains: "", is: "", text: "cocoon"}, + {contains: "kike", is: "kike", text: "kike"}, + {contains: "nigger", is: "nigger", text: "nigger"}, + {contains: "nigger", is: "nigger", text: "niggers"}, + {contains: "nigger", is: "nigger", text: "n1gg4"}, + {contains: "nigger", is: "nigger", text: "niggas"}, + {contains: "", is: "", text: "niggle"}, + {contains: "", is: "", text: "niggling"}, + {contains: "", is: "", text: "snigger"}, + {contains: "tranny", is: "tranny", text: "tranny"}, + {contains: "tranny", is: "tranny", text: "trannie"}, + {contains: "tranny", is: "", text: "blahtrannie"}, + } + + for _, fix := range fixtures { + assert.Equal(fix.contains, SlugContainsExplicitSlur(fix.text)) + assert.Equal(fix.is, SlugIsExplicitSlur(fix.text)) + } +} + +func TestStringContainsExplicitSlur(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + text string + out string + }{ + {out: "", text: ""}, + {out: "", text: "hello"}, + {out: "chink", text: "CHINK"}, + {out: "faggot", text: "f-a-g-g-o-t"}, + {out: "faggot", text: "f a g g o t"}, + {out: "faggot", text: "f\na\ng\ng\no\nt"}, + {out: "kike", text: "kike"}, + {out: "nigger", text: "niggers"}, + {out: "nigger", text: "niggers.bsky.social"}, + {out: "nigger", text: "pumpkinniggah.bsky.social"}, + {out: "tranny", text: "trannie"}, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, SlugContainsExplicitSlur(Slugify(fix.text))) + } +} diff --git a/automod/keyword/tokenize.go b/automod/keyword/tokenize.go new file mode 100644 index 000000000..0b5a33ca4 --- /dev/null +++ b/automod/keyword/tokenize.go @@ -0,0 +1,61 @@ +package keyword + +import ( + "log/slog" + "regexp" + "strings" + "unicode" + + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +var ( + puncChars = regexp.MustCompile(`[[:punct:]]+`) + nonTokenChars = regexp.MustCompile(`[^\pL\pN\s]+`) + nonTokenCharsSkipCensorChars = regexp.MustCompile(`[^\pL\pN\s#*_-]`) +) + +// Splits free-form text in to tokens, including lower-case, unicode normalization, and some unicode folding. +// +// The intent is for this to work similarly to an NLP tokenizer, as might be used in a fulltext search engine, and enable fast matching to a list of known tokens. It might eventually even do stemming, removing pluralization (trailing "s" for English), etc. +func TokenizeTextWithRegex(text string, nonTokenCharsRegex *regexp.Regexp) []string { + // this function needs to be re-defined in every function call to prevent a race condition + normFunc := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) + split := strings.ToLower(nonTokenCharsRegex.ReplaceAllString(text, " ")) + bare := strings.ToLower(nonTokenCharsRegex.ReplaceAllString(split, "")) + norm, _, err := transform.String(normFunc, bare) + if err != nil { + slog.Warn("unicode normalization error", "err", err) + norm = bare + } + return strings.Fields(norm) +} + +func TokenizeText(text string) []string { + return TokenizeTextWithRegex(text, nonTokenChars) +} + +func TokenizeTextSkippingCensorChars(text string) []string { + return TokenizeTextWithRegex(text, nonTokenCharsSkipCensorChars) +} + +func splitIdentRune(c rune) bool { + return !unicode.IsLetter(c) && !unicode.IsNumber(c) +} + +// Splits an identifier in to tokens. Removes any single-character tokens. +// +// For example, the-handle.bsky.social would be split in to ["the", "handle", "bsky", "social"] +func TokenizeIdentifier(orig string) []string { + fields := strings.FieldsFunc(orig, splitIdentRune) + out := make([]string, 0, len(fields)) + for _, v := range fields { + tok := Slugify(v) + if len(tok) > 1 { + out = append(out, tok) + } + } + return out +} diff --git a/automod/keyword/tokenize_test.go b/automod/keyword/tokenize_test.go new file mode 100644 index 000000000..45b477f6d --- /dev/null +++ b/automod/keyword/tokenize_test.go @@ -0,0 +1,89 @@ +package keyword + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTokenizeText(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + text string + out []string + }{ + {text: "", out: []string{}}, + {text: "Hello, โลก!", out: []string{"hello", "โลก"}}, + {text: "Gdańsk", out: []string{"gdansk"}}, + {text: " foo1;bar2,baz3...", out: []string{"foo1", "bar2", "baz3"}}, + {text: "foo*bar", out: []string{"foo", "bar"}}, + {text: "foo-bar", out: []string{"foo", "bar"}}, + {text: "foo_bar", out: []string{"foo", "bar"}}, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, TokenizeText(fix.text)) + } +} + +func TestTokenizeTextWithCensorChars(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + text string + out []string + }{ + {text: "", out: []string{}}, + {text: "Hello, โลก!", out: []string{"hello", "โลก"}}, + {text: "Gdańsk", out: []string{"gdansk"}}, + {text: " foo1;bar2,baz3...", out: []string{"foo1", "bar2", "baz3"}}, + {text: "foo*bar,foo&bar", out: []string{"foo*bar", "foo", "bar"}}, + {text: "foo-bar,foo&bar", out: []string{"foo-bar", "foo", "bar"}}, + {text: "foo_bar,foo&bar", out: []string{"foo_bar", "foo", "bar"}}, + {text: "foo#bar,foo&bar", out: []string{"foo#bar", "foo", "bar"}}, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, TokenizeTextSkippingCensorChars(fix.text)) + } +} + +func TestTokenizeTextWithCustomRegex(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + text string + out []string + }{ + {text: "", out: []string{}}, + {text: "Hello, โลก!", out: []string{"hello", "โลก"}}, + {text: "Gdańsk", out: []string{"gdansk"}}, + {text: " foo1;bar2,baz3...", out: []string{"foo1", "bar2", "baz3"}}, + {text: "foo*bar", out: []string{"foo", "bar"}}, + {text: "foo&bar,foo*bar", out: []string{"foo&bar", "foo", "bar"}}, + } + + regex := regexp.MustCompile(`[^\pL\pN\s&]`) + for _, fix := range fixtures { + assert.Equal(fix.out, TokenizeTextWithRegex(fix.text, regex)) + } +} + +func TestTokenizeIdentifier(t *testing.T) { + assert := assert.New(t) + + fixtures := []struct { + ident string + out []string + }{ + {ident: "", out: []string{}}, + {ident: "the-handle.example.com", out: []string{"the", "handle", "example", "com"}}, + {ident: "@a-b-c", out: []string{}}, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, TokenizeIdentifier(fix.ident)) + } +} diff --git a/automod/pkg.go b/automod/pkg.go new file mode 100644 index 000000000..7d5faa068 --- /dev/null +++ b/automod/pkg.go @@ -0,0 +1,45 @@ +package automod + +import ( + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/engine" +) + +type Engine = engine.Engine +type EngineConfig = engine.EngineConfig +type AccountMeta = engine.AccountMeta +type ProfileSummary = engine.ProfileSummary +type AccountPrivate = engine.AccountPrivate +type RuleSet = engine.RuleSet + +type Notifier = engine.Notifier +type SlackNotifier = engine.SlackNotifier + +type AccountContext = engine.AccountContext +type RecordContext = engine.RecordContext +type OzoneEventContext = engine.OzoneEventContext +type RecordOp = engine.RecordOp + +type IdentityRuleFunc = engine.IdentityRuleFunc +type RecordRuleFunc = engine.RecordRuleFunc +type PostRuleFunc = engine.PostRuleFunc +type ProfileRuleFunc = engine.ProfileRuleFunc +type BlobRuleFunc = engine.BlobRuleFunc +type OzoneEventRuleFunc = engine.OzoneEventRuleFunc + +var ( + ReportReasonSpam = engine.ReportReasonSpam + ReportReasonViolation = engine.ReportReasonViolation + ReportReasonMisleading = engine.ReportReasonMisleading + ReportReasonSexual = engine.ReportReasonSexual + ReportReasonRude = engine.ReportReasonRude + ReportReasonOther = engine.ReportReasonOther + + PeriodTotal = countstore.PeriodTotal + PeriodDay = countstore.PeriodDay + PeriodHour = countstore.PeriodHour + + CreateOp = engine.CreateOp + UpdateOp = engine.UpdateOp + DeleteOp = engine.DeleteOp +) diff --git a/automod/rules/all.go b/automod/rules/all.go new file mode 100644 index 000000000..facb503dd --- /dev/null +++ b/automod/rules/all.go @@ -0,0 +1,64 @@ +package rules + +import ( + "github.com/bluesky-social/indigo/automod" +) + +// IMPORTANT: reminder that these are the indigo-edition rules, not production rules +func DefaultRules() automod.RuleSet { + rules := automod.RuleSet{ + PostRules: []automod.PostRuleFunc{ + //MisleadingURLPostRule, + //MisleadingMentionPostRule, + ReplyCountPostRule, + YoungAccountDistinctRepliesRule, + BadHashtagsPostRule, + //TooManyHashtagsPostRule, + //AccountDemoPostRule, + AccountPrivateDemoPostRule, + GtubePostRule, + BadWordPostRule, + ReplySingleBadWordPostRule, + AggressivePromotionRule, + IdenticalReplyPostRule, + //IdenticalReplyPostSameParentRule, + DistinctMentionsRule, + YoungAccountDistinctMentionsRule, + MisleadingLinkUnicodeReversalPostRule, + SimpleBotPostRule, + HarassmentTargetInteractionPostRule, + HarassmentTrivialPostRule, + NostrSpamPostRule, + TrivialSpamPostRule, + }, + ProfileRules: []automod.ProfileRuleFunc{ + GtubeProfileRule, + BadWordProfileRule, + BotLinkProfileRule, + CelebSpamProfileRule, + }, + RecordRules: []automod.RecordRuleFunc{ + InteractionChurnRule, + BadWordRecordKeyRule, + BadWordOtherRecordRule, + TooManyRepostRule, + }, + RecordDeleteRules: []automod.RecordRuleFunc{ + DeleteInteractionRule, + }, + IdentityRules: []automod.IdentityRuleFunc{ + NewAccountRule, + BadWordHandleRule, + BadWordDIDRule, + NewAccountBotEmailRule, + CelebSpamIdentityRule, + }, + BlobRules: []automod.BlobRuleFunc{ + //BlobVerifyRule, + }, + OzoneEventRules: []automod.OzoneEventRuleFunc{ + HarassmentProtectionOzoneEventRule, + }, + } + return rules +} diff --git a/automod/rules/blobs.go b/automod/rules/blobs.go new file mode 100644 index 000000000..9ebbeadc8 --- /dev/null +++ b/automod/rules/blobs.go @@ -0,0 +1,24 @@ +package rules + +import ( + "github.com/bluesky-social/indigo/automod" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +var _ automod.BlobRuleFunc = BlobVerifyRule + +func BlobVerifyRule(c *automod.RecordContext, blob lexutil.LexBlob, data []byte) error { + + if len(data) == 0 { + c.AddRecordFlag("empty-blob") + } + + // check size + if blob.Size >= 0 && int64(len(data)) != blob.Size { + c.AddRecordFlag("invalid-blob") + } else { + c.Logger.Info("blob checks out", "cid", blob.Ref, "size", blob.Size, "mimetype", blob.MimeType) + } + + return nil +} diff --git a/automod/rules/doc.go b/automod/rules/doc.go new file mode 100644 index 000000000..95fa57de3 --- /dev/null +++ b/automod/rules/doc.go @@ -0,0 +1,2 @@ +// Example automod rules and helpers. +package rules diff --git a/automod/rules/example_sets.json b/automod/rules/example_sets.json new file mode 100644 index 000000000..1d741a389 --- /dev/null +++ b/automod/rules/example_sets.json @@ -0,0 +1,21 @@ +{ + "bad-hashtags": [ + "deathtooutgroup" + ], + "bad-words": [ + "hardar", + "veryhardar" + ], + "worst-words": [ + "veryhardar" + ], + "harassment-target-dids": [ + "did:web:harassed.example.com" + ], + "promo-domain": [ + "buy-crypto.example.com" + ], + "trivial-spam-text": [ + "spam" + ] +} diff --git a/automod/rules/gtube.go b/automod/rules/gtube.go new file mode 100644 index 000000000..71922a528 --- /dev/null +++ b/automod/rules/gtube.go @@ -0,0 +1,33 @@ +package rules + +import ( + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" +) + +// https://en.wikipedia.org/wiki/GTUBE +var gtubeString = "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" + +var _ automod.PostRuleFunc = GtubePostRule + +func GtubePostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if strings.Contains(post.Text, gtubeString) { + c.AddRecordLabel("spam") + c.Notify("slack") + c.AddRecordTag("gtube-record") + } + return nil +} + +var _ automod.ProfileRuleFunc = GtubeProfileRule + +func GtubeProfileRule(c *automod.RecordContext, profile *appbsky.ActorProfile) error { + if profile.Description != nil && strings.Contains(*profile.Description, gtubeString) { + c.AddRecordLabel("spam") + c.Notify("slack") + c.AddAccountTag("gtuber-account") + } + return nil +} diff --git a/automod/rules/harassment.go b/automod/rules/harassment.go new file mode 100644 index 000000000..e38e7c44a --- /dev/null +++ b/automod/rules/harassment.go @@ -0,0 +1,158 @@ +package rules + +import ( + "fmt" + "time" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" +) + +var _ automod.PostRuleFunc = HarassmentTargetInteractionPostRule + +// looks for new accounts, which interact with frequently-harassed accounts, and report them for review +func HarassmentTargetInteractionPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 24*time.Hour) { + return nil + } + + var interactionDIDs []string + facets, err := helpers.ExtractFacets(post) + if err != nil { + return err + } + for _, pf := range facets { + if pf.DID != nil { + interactionDIDs = append(interactionDIDs, *pf.DID) + } + } + if post.Reply != nil && !helpers.IsSelfThread(c, post) { + parentURI, err := syntax.ParseATURI(post.Reply.Parent.Uri) + if err != nil { + return err + } + interactionDIDs = append(interactionDIDs, parentURI.Authority().String()) + } + // quote posts + if post.Embed != nil && post.Embed.EmbedRecord != nil && post.Embed.EmbedRecord.Record != nil { + uri, err := syntax.ParseATURI(post.Embed.EmbedRecord.Record.Uri) + if err != nil { + c.Logger.Warn("invalid AT-URI in post embed record (quote-post)", "uri", post.Embed.EmbedRecord.Record.Uri) + } else { + interactionDIDs = append(interactionDIDs, uri.Authority().String()) + } + } + if len(interactionDIDs) == 0 { + return nil + } + + // more than a handful of followers or posts from author account? skip + if c.Account.FollowersCount > 10 || c.Account.PostsCount > 10 { + return nil + } + postCount := c.GetCount("post", c.Account.Identity.DID.String(), countstore.PeriodTotal) + if postCount > 20 { + return nil + } + + interactionDIDs = helpers.DedupeStrings(interactionDIDs) + for _, d := range interactionDIDs { + did, err := syntax.ParseDID(d) + if err != nil { + c.Logger.Warn("invalid DID in record", "did", d) + continue + } + if did == c.Account.Identity.DID { + continue + } + targetIsProtected := false + if c.InSet("harassment-target-dids", did.String()) { + targetIsProtected = true + } else { + // check if the target account has a harassment protection tag in Ozone + targetAccount := c.GetAccountMeta(did) + if targetAccount == nil { + continue + } + if targetAccount.Private != nil { + for _, t := range targetAccount.Private.AccountTags { + if t == "harassment-protection" { + targetIsProtected = true + break + } + } + } + } + + if !targetIsProtected { + continue + } + + // ignore if the target account follows the new account + rel := c.GetAccountRelationship(syntax.DID(did)) + if rel.FollowedBy { + continue + } + + //c.AddRecordFlag("interaction-harassed-target") + var privCreatedAt *time.Time + if c.Account.Private != nil && c.Account.Private.IndexedAt != nil { + privCreatedAt = c.Account.Private.IndexedAt + } + c.Logger.Warn("possible harassment", "targetDID", did, "author", c.Account.Identity.DID, "accountCreated", c.Account.CreatedAt, "privateAccountCreated", privCreatedAt) + c.ReportAccount(automod.ReportReasonOther, fmt.Sprintf("possible harassment of known target account: %s (also labeled; remove label if this isn't harassment)", did)) + c.AddAccountLabel("!hide") + c.Notify("slack") + return nil + } + return nil +} + +var _ automod.PostRuleFunc = HarassmentTrivialPostRule + +// looks for new accounts, which frequently post the same type of content +func HarassmentTrivialPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { + return nil + } + + // only posts with dumb pattern + if post.Text != "F" { + return nil + } + + did := c.Account.Identity.DID.String() + c.Increment("trivial-harassing", did) + count := c.GetCount("trivial-harassing", did, countstore.PeriodDay) + + if count > 5 { + //c.AddRecordFlag("trivial-harassing-post") + c.ReportAccount(automod.ReportReasonOther, "possible targetted harassment (also labeled; remove label if this isn't harassment!)") + c.AddAccountLabel("!hide") + c.Notify("slack") + } + return nil +} + +var _ automod.OzoneEventRuleFunc = HarassmentProtectionOzoneEventRule + +// looks for new harassment protection tags on accounts, and logs them +func HarassmentProtectionOzoneEventRule(c *automod.OzoneEventContext) error { + if c.Event.EventType != "tag" || c.Event.Event.ModerationDefs_ModEventTag == nil { + return nil + } + + for _, t := range c.Event.Event.ModerationDefs_ModEventTag.Add { + if t == "harassment-protection" { + c.Logger.Info("adding harassment protection to account", "ozoneComment", c.Event.Event.ModerationDefs_ModEventTag.Comment, "did", c.Account.Identity.DID, "handle", c.Account.Identity.Handle) + // to make slack message clearer; bluring flags and tags is a bit weird + c.AddAccountFlag("harassment-protection") + //c.Notify("slack") + break + } + } + return nil +} diff --git a/automod/rules/hashtags.go b/automod/rules/hashtags.go new file mode 100644 index 000000000..682ce746a --- /dev/null +++ b/automod/rules/hashtags.go @@ -0,0 +1,56 @@ +package rules + +import ( + "fmt" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" + "github.com/bluesky-social/indigo/automod/keyword" +) + +// looks for specific hashtags from known lists +func BadHashtagsPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + for _, tag := range helpers.ExtractHashtagsPost(post) { + tag = helpers.NormalizeHashtag(tag) + // skip some bad-word hashtags which frequently false-positive + if tag == "nazi" || tag == "hitler" { + continue + } + if c.InSet("bad-hashtags", tag) || c.InSet("bad-words", tag) { + c.AddRecordFlag("bad-hashtag") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in hashtags: %s", tag)) + break + } + word := keyword.SlugContainsExplicitSlur(keyword.Slugify(tag)) + if word != "" { + c.AddAccountFlag("bad-hashtag") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in hashtags: %s", word)) + break + } + } + return nil +} + +var _ automod.PostRuleFunc = BadHashtagsPostRule + +// if a post is "almost all" hashtags, it might be a form of search spam +func TooManyHashtagsPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + tags := helpers.ExtractHashtagsPost(post) + tagChars := 0 + for _, tag := range tags { + tagChars += len(tag) + } + tagTextRatio := float64(tagChars) / float64(len(post.Text)) + // if there is an image, allow some more tags + if len(tags) > 4 && tagTextRatio > 0.6 && post.Embed.EmbedImages == nil { + c.AddRecordFlag("many-hashtags") + c.Notify("slack") + } else if len(tags) > 7 && tagTextRatio > 0.8 { + c.AddRecordFlag("many-hashtags") + c.Notify("slack") + } + return nil +} + +var _ automod.PostRuleFunc = TooManyHashtagsPostRule diff --git a/automod/rules/hashtags_test.go b/automod/rules/hashtags_test.go new file mode 100644 index 000000000..e7c261d4a --- /dev/null +++ b/automod/rules/hashtags_test.go @@ -0,0 +1,60 @@ +package rules + +import ( + "bytes" + "context" + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/engine" + + "github.com/stretchr/testify/assert" +) + +func TestBadHashtagPostRule(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + eng := engine.EngineTestFixture() + am1 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + } + cid1 := syntax.CID("cid123") + p1 := appbsky.FeedPost{ + Text: "some post blah", + } + p1buf := new(bytes.Buffer) + assert.NoError(p1.MarshalCBOR(p1buf)) + p1cbor := p1buf.Bytes() + op := engine.RecordOp{ + Action: engine.CreateOp, + DID: am1.Identity.DID, + Collection: syntax.NSID("app.bsky.feed.post"), + RecordKey: syntax.RecordKey("abc123"), + CID: &cid1, + RecordCBOR: p1cbor, + } + c1 := engine.NewRecordContext(ctx, &eng, am1, op) + assert.NoError(BadHashtagsPostRule(&c1, &p1)) + eff1 := engine.ExtractEffects(&c1.BaseContext) + assert.Empty(eff1.RecordFlags) + + p2 := appbsky.FeedPost{ + Text: "some post blah", + Tags: []string{"one", "slur"}, + } + p2buf := new(bytes.Buffer) + assert.NoError(p2.MarshalCBOR(p2buf)) + p2cbor := p2buf.Bytes() + op.RecordCBOR = p2cbor + c2 := engine.NewRecordContext(ctx, &eng, am1, op) + assert.NoError(BadHashtagsPostRule(&c2, &p2)) + eff2 := engine.ExtractEffects(&c2.BaseContext) + assert.NotEmpty(eff2.RecordFlags) +} diff --git a/automod/rules/identity.go b/automod/rules/identity.go new file mode 100644 index 000000000..e74991233 --- /dev/null +++ b/automod/rules/identity.go @@ -0,0 +1,59 @@ +package rules + +import ( + "net/url" + "strings" + "time" + + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" +) + +// triggers on first identity event for an account (DID) +func NewAccountRule(c *automod.AccountContext) error { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(c, 4*time.Hour) { + return nil + } + + did := c.Account.Identity.DID.String() + exists := c.GetCount("acct/exists", did, countstore.PeriodTotal) + if exists == 0 { + c.Logger.Info("new account") + c.Increment("acct/exists", did) + + pdsURL, err := url.Parse(c.Account.Identity.PDSEndpoint()) + if err != nil { + c.Logger.Warn("invalid PDS URL", "err", err, "endpoint", c.Account.Identity.PDSEndpoint()) + return nil + } + pdsHost := strings.ToLower(pdsURL.Host) + existingAccounts := c.GetCount("host/newacct", pdsHost, countstore.PeriodTotal) + c.Increment("host/newacct", pdsHost) + + // new PDS host + if existingAccounts == 0 { + c.Logger.Info("new PDS instance", "pdsHost", pdsHost) + c.Increment("host", "new") + c.AddAccountFlag("host-first-account") + c.Notify("slack") + } + } + return nil +} + +var _ automod.IdentityRuleFunc = NewAccountRule + +func CelebSpamIdentityRule(c *automod.AccountContext) error { + + hdl := c.Account.Identity.Handle.String() + if strings.Contains(hdl, "elon") && strings.Contains(hdl, "musk") { + c.AddAccountFlag("handle-elon-musk") + c.ReportAccount(automod.ReportReasonSpam, "possible Elon Musk impersonator") + return nil + } + + return nil +} + +var _ automod.IdentityRuleFunc = CelebSpamIdentityRule diff --git a/automod/rules/interaction.go b/automod/rules/interaction.go new file mode 100644 index 000000000..f8416dc46 --- /dev/null +++ b/automod/rules/interaction.go @@ -0,0 +1,69 @@ +package rules + +import ( + "fmt" + + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/countstore" +) + +var interactionDailyThreshold = 800 +var followsDailyThreshold = 3000 + +var _ automod.RecordRuleFunc = InteractionChurnRule + +// looks for accounts which do frequent interaction churn, such as follow-unfollow. +func InteractionChurnRule(c *automod.RecordContext) error { + + did := c.Account.Identity.DID.String() + switch c.RecordOp.Collection { + case "app.bsky.feed.like": + c.Increment("like", did) + created := c.GetCount("like", did, countstore.PeriodDay) + deleted := c.GetCount("unlike", did, countstore.PeriodDay) + ratio := float64(deleted) / float64(created) + if created > interactionDailyThreshold && deleted > interactionDailyThreshold && ratio > 0.5 { + c.Logger.Info("high-like-churn", "created-today", created, "deleted-today", deleted) + c.AddAccountFlag("high-like-churn") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("interaction churn: %d likes, %d unlikes today (so far)", created, deleted)) + // c.EscalateAccount() + c.Notify("slack") + return nil + } + case "app.bsky.graph.follow": + c.Increment("follow", did) + created := c.GetCount("follow", did, countstore.PeriodDay) + deleted := c.GetCount("unfollow", did, countstore.PeriodDay) + ratio := float64(deleted) / float64(created) + if created > interactionDailyThreshold && deleted > interactionDailyThreshold && ratio > 0.5 { + c.Logger.Info("high-follow-churn", "created-today", created, "deleted-today", deleted) + c.AddAccountFlag("high-follow-churn") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("interaction churn: %d follows, %d unfollows today (so far)", created, deleted)) + // c.EscalateAccount() + c.Notify("slack") + return nil + } + // just generic bulk following + if created > followsDailyThreshold { + c.Logger.Info("bulk-follower", "created-today", created) + c.AddAccountFlag("bulk-follower") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("bulk following: %d follows today (so far)", created)) + c.Notify("slack") + return nil + } + } + return nil +} + +var _ automod.RecordRuleFunc = DeleteInteractionRule + +func DeleteInteractionRule(c *automod.RecordContext) error { + did := c.Account.Identity.DID.String() + switch c.RecordOp.Collection { + case "app.bsky.feed.like": + c.Increment("unlike", did) + case "app.bsky.graph.follow": + c.Increment("unfollow", did) + } + return nil +} diff --git a/automod/rules/keyword.go b/automod/rules/keyword.go new file mode 100644 index 000000000..8d5caa395 --- /dev/null +++ b/automod/rules/keyword.go @@ -0,0 +1,235 @@ +package rules + +import ( + "bytes" + "fmt" + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" + "github.com/bluesky-social/indigo/automod/keyword" +) + +func BadWordPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + isJapanese := false + for _, lang := range post.Langs { + if lang == "ja" || strings.HasPrefix(lang, "ja-") { + isJapanese = true + } + } + for _, tok := range helpers.ExtractTextTokensPost(post) { + word := keyword.SlugIsExplicitSlur(tok) + // used very frequently in a reclaimed context + if word != "" && word != "faggot" && word != "tranny" && word != "coon" && !(word == "kike" && isJapanese) { + c.AddRecordFlag("bad-word-text") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in post text or alttext: %s", word)) + //c.Notify("slack") + break + } + // de-pluralize + tok = strings.TrimSuffix(tok, "s") + if c.InSet("worst-words", tok) { + // skip this specific term, if used in a Japanese language post + if isJapanese && tok == "kike" { + continue + } + + c.AddRecordFlag("bad-word-text") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in post text or alttext: %s", tok)) + //c.Notify("slack") + break + } + } + return nil +} + +var _ automod.PostRuleFunc = BadWordPostRule + +func BadWordProfileRule(c *automod.RecordContext, profile *appbsky.ActorProfile) error { + if profile.DisplayName != nil { + word := keyword.SlugContainsExplicitSlur(keyword.Slugify(*profile.DisplayName)) + if word != "" { + c.AddRecordFlag("bad-word-name") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in display name: %s", word)) + //c.Notify("slack") + } + } + for _, tok := range helpers.ExtractTextTokensProfile(profile) { + // de-pluralize + tok = strings.TrimSuffix(tok, "s") + if c.InSet("worst-words", tok) { + c.AddRecordFlag("bad-word-text") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in profile description: %s", tok)) + //c.Notify("slack") + break + } + } + return nil +} + +var _ automod.ProfileRuleFunc = BadWordProfileRule + +// looks for the specific harassment situation of a replay to another user with only a single word +func ReplySingleBadWordPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if post.Reply != nil && !helpers.IsSelfThread(c, post) { + tokens := helpers.ExtractTextTokensPost(post) + if len(tokens) != 1 { + return nil + } + tok := tokens[0] + if c.InSet("bad-words", tok) || keyword.SlugIsExplicitSlur(tok) != "" { + c.AddRecordFlag("reply-single-bad-word") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("bad single-word reply: %s", tok)) + //c.Notify("slack") + } + } + return nil +} + +var _ automod.PostRuleFunc = ReplySingleBadWordPostRule + +// scans for bad keywords in records other than posts and profiles +func BadWordOtherRecordRule(c *automod.RecordContext) error { + name := "" + text := "" + switch c.RecordOp.Collection.String() { + case "app.bsky.graph.list": + var list appbsky.GraphList + if err := list.UnmarshalCBOR(bytes.NewReader(c.RecordOp.RecordCBOR)); err != nil { + return fmt.Errorf("failed to parse app.bsky.graph.list record: %v", err) + } + name += " " + list.Name + if list.Description != nil { + text += " " + *list.Description + } + if list.Purpose != nil { + text += " " + *list.Purpose + } + case "app.bsky.feed.generator": + var generator appbsky.FeedGenerator + if err := generator.UnmarshalCBOR(bytes.NewReader(c.RecordOp.RecordCBOR)); err != nil { + return fmt.Errorf("failed to parse app.bsky.feed.generator record: %v", err) + } + name += " " + generator.DisplayName + if generator.Description != nil { + text += " " + *generator.Description + } + } + if name != "" { + // check for explicit slurs or bad word tokens + word := keyword.SlugContainsExplicitSlur(keyword.Slugify(name)) + if word != "" { + c.AddRecordFlag("bad-word-name") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in name: %s", word)) + c.Notify("slack") + } + tokens := keyword.TokenizeText(name) + for _, tok := range tokens { + if c.InSet("bad-words", tok) { + c.AddRecordFlag("bad-word-name") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in name: %s", tok)) + c.Notify("slack") + break + } + } + } + if text != "" { + // check for explicit slurs or worst word tokens + word := keyword.SlugContainsExplicitSlur(keyword.Slugify(text)) + if word != "" { + c.AddRecordFlag("bad-word-text") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in description: %s", word)) + c.Notify("slack") + } + tokens := keyword.TokenizeText(text) + for _, tok := range tokens { + // de-pluralize + tok = strings.TrimSuffix(tok, "s") + if c.InSet("worst-words", tok) { + c.AddRecordFlag("bad-word-text") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in description: %s", tok)) + c.Notify("slack") + break + } + } + } + return nil +} + +var _ automod.RecordRuleFunc = BadWordOtherRecordRule + +// scans the record-key for all records +func BadWordRecordKeyRule(c *automod.RecordContext) error { + // check record key + word := keyword.SlugIsExplicitSlur(keyword.Slugify(c.RecordOp.RecordKey.String())) + if word != "" { + c.AddRecordFlag("bad-word-recordkey") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in record-key (URL): %s", word)) + c.Notify("slack") + } + tokens := keyword.TokenizeIdentifier(c.RecordOp.RecordKey.String()) + for _, tok := range tokens { + if c.InSet("bad-words", tok) { + c.AddRecordFlag("bad-word-recordkey") + c.ReportRecord(automod.ReportReasonRude, fmt.Sprintf("possible bad word in record-key (URL): %s", tok)) + c.Notify("slack") + break + } + } + + return nil +} + +var _ automod.RecordRuleFunc = BadWordRecordKeyRule + +func BadWordHandleRule(c *automod.AccountContext) error { + word := keyword.SlugContainsExplicitSlur(keyword.Slugify(c.Account.Identity.Handle.String())) + if word != "" { + c.AddAccountFlag("bad-word-handle") + c.ReportAccount(automod.ReportReasonRude, fmt.Sprintf("possible bad word in handle (username): %s", word)) + //c.Notify("slack") + return nil + } + + tokens := keyword.TokenizeIdentifier(c.Account.Identity.Handle.String()) + for _, tok := range tokens { + if c.InSet("bad-words", tok) { + c.AddAccountFlag("bad-word-handle") + c.ReportAccount(automod.ReportReasonRude, fmt.Sprintf("possible bad word in handle (username): %s", tok)) + //c.Notify("slack") + break + } + } + + return nil +} + +var _ automod.IdentityRuleFunc = BadWordHandleRule + +func BadWordDIDRule(c *automod.AccountContext) error { + if c.Account.Identity.DID.Method() == "plc" { + return nil + } + word := keyword.SlugContainsExplicitSlur(keyword.Slugify(c.Account.Identity.DID.String())) + if word != "" { + c.AddAccountFlag("bad-word-did") + c.ReportAccount(automod.ReportReasonRude, fmt.Sprintf("possible bad word in DID (account identifier): %s", word)) + c.Notify("slack") + return nil + } + + tokens := keyword.TokenizeIdentifier(c.Account.Identity.DID.String()) + for _, tok := range tokens { + if c.InSet("bad-words", tok) { + c.AddAccountFlag("bad-word-did") + c.ReportAccount(automod.ReportReasonRude, fmt.Sprintf("possible bad word in DID (account identifier): %s", tok)) + c.Notify("slack") + break + } + } + + return nil +} + +var _ automod.IdentityRuleFunc = BadWordDIDRule diff --git a/automod/rules/keyword_test.go b/automod/rules/keyword_test.go new file mode 100644 index 000000000..465bdbb4e --- /dev/null +++ b/automod/rules/keyword_test.go @@ -0,0 +1,109 @@ +package rules + +import ( + "bytes" + "context" + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/engine" + + "github.com/stretchr/testify/assert" +) + +func TestBadWordHandleRule(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + eng := engine.EngineTestFixture() + am1 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + } + am2 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc222"), + Handle: syntax.Handle("hardr.example.com"), + }, + } + am3 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc333"), + Handle: syntax.Handle("f.agg.ot"), + }, + } + + ac1 := engine.NewAccountContext(ctx, &eng, am1) + assert.NoError(BadWordHandleRule(&ac1)) + eff1 := engine.ExtractEffects(&ac1.BaseContext) + assert.Empty(eff1.RecordFlags) + + ac2 := engine.NewAccountContext(ctx, &eng, am2) + assert.NoError(BadWordHandleRule(&ac2)) + eff2 := engine.ExtractEffects(&ac2.BaseContext) + assert.Equal([]string{"bad-word-handle"}, eff2.AccountFlags) + + ac3 := engine.NewAccountContext(ctx, &eng, am3) + assert.NoError(BadWordHandleRule(&ac3)) + eff3 := engine.ExtractEffects(&ac3.BaseContext) + assert.Equal([]string{"bad-word-handle"}, eff3.AccountFlags) +} + +func TestBadWordPostRule(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + eng := engine.EngineTestFixture() + am1 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + } + + // record key + cid1 := syntax.CID("cid123") + p1 := appbsky.FeedPost{ + Text: "some post blah", + } + p1buf := new(bytes.Buffer) + assert.NoError(p1.MarshalCBOR(p1buf)) + p1cbor := p1buf.Bytes() + op := engine.RecordOp{ + Action: engine.CreateOp, + DID: am1.Identity.DID, + Collection: syntax.NSID("app.bsky.feed.post"), + RecordKey: syntax.RecordKey("fagg0t"), + CID: &cid1, + RecordCBOR: p1cbor, + } + c1 := engine.NewRecordContext(ctx, &eng, am1, op) + assert.NoError(BadWordRecordKeyRule(&c1)) + eff1 := engine.ExtractEffects(&c1.BaseContext) + assert.Equal([]string{"bad-word-recordkey"}, eff1.RecordFlags) + + // token in body + p2 := appbsky.FeedPost{ + Text: "some post hardestr blah", + } + p2buf := new(bytes.Buffer) + assert.NoError(p2.MarshalCBOR(p2buf)) + p2cbor := p2buf.Bytes() + op2 := engine.RecordOp{ + Action: engine.CreateOp, + DID: am1.Identity.DID, + Collection: syntax.NSID("app.bsky.feed.post"), + RecordKey: syntax.RecordKey("abc123"), + CID: &cid1, + RecordCBOR: p2cbor, + } + c2 := engine.NewRecordContext(ctx, &eng, am1, op2) + assert.NoError(BadWordPostRule(&c2, &p2)) + eff2 := engine.ExtractEffects(&c2.BaseContext) + assert.Equal([]string{"bad-word-text"}, eff2.RecordFlags) +} diff --git a/automod/rules/mentions.go b/automod/rules/mentions.go new file mode 100644 index 000000000..98d419d09 --- /dev/null +++ b/automod/rules/mentions.go @@ -0,0 +1,95 @@ +package rules + +import ( + "fmt" + "time" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" +) + +var _ automod.PostRuleFunc = DistinctMentionsRule + +var mentionHourlyThreshold = 40 + +// DistinctMentionsRule looks for accounts which mention an unusually large number of distinct accounts per period. +func DistinctMentionsRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + did := c.Account.Identity.DID.String() + + // Increment counters for all new mentions in this post. + var newMentions bool + for _, facet := range post.Facets { + for _, feature := range facet.Features { + mention := feature.RichtextFacet_Mention + if mention == nil { + continue + } + c.IncrementDistinct("mentions", did, mention.Did) + newMentions = true + } + } + + // If there were any new mentions, check if it's gotten spammy. + if !newMentions { + return nil + } + if mentionHourlyThreshold <= c.GetCountDistinct("mentions", did, countstore.PeriodHour) { + c.AddAccountFlag("high-distinct-mentions") + c.Notify("slack") + } + + return nil +} + +var youngMentionAccountLimit = 12 +var _ automod.PostRuleFunc = YoungAccountDistinctMentionsRule + +func YoungAccountDistinctMentionsRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 14*24*time.Hour) { + return nil + } + + // parse out all the mentions + var mentionedAccounts []syntax.DID + for _, facet := range post.Facets { + for _, feature := range facet.Features { + mention := feature.RichtextFacet_Mention + if mention == nil { + continue + } + did, err := syntax.ParseDID(mention.Did) + if err != nil { + continue + } + mentionedAccounts = append(mentionedAccounts, did) + } + } + if len(mentionedAccounts) == 0 { + return nil + } + + did := c.Account.Identity.DID.String() + + // check for relationships, and increment accounts + newMentions := 0 + for _, otherDID := range mentionedAccounts { + rel := c.GetAccountRelationship(otherDID) + if rel.FollowedBy { + continue + } + c.IncrementDistinct("young-mention", did, otherDID.String()) + newMentions += 1 + } + + count := c.GetCountDistinct("young-mention", did, countstore.PeriodHour) + newMentions + if count >= youngMentionAccountLimit { + c.AddAccountFlag("new-account-distinct-account-mention") + c.ReportAccount(automod.ReportReasonRude, fmt.Sprintf("possible spam (new account, mentioned %d distinct accounts in past hour)", count)) + c.Notify("slack") + } + + return nil +} diff --git a/automod/rules/misleading.go b/automod/rules/misleading.go new file mode 100644 index 000000000..31822ccce --- /dev/null +++ b/automod/rules/misleading.go @@ -0,0 +1,146 @@ +package rules + +import ( + "log/slog" + "net/url" + "strings" + "unicode" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" +) + +func isMisleadingURLFacet(facet helpers.PostFacet, logger *slog.Logger) bool { + linkURL, err := url.Parse(*facet.URL) + if err != nil { + logger.Warn("invalid link metadata URL", "url", facet.URL) + return false + } + + // basic text string pre-cleanups + text := strings.ToLower(strings.TrimSpace(facet.Text)) + + // remove square brackets + if strings.HasPrefix(text, "[") && strings.HasSuffix(text, "]") { + text = text[1 : len(text)-1] + } + + // truncated and not an obvious prefix hack (TODO: more special domains? regex?) + if strings.HasSuffix(text, "...") && !strings.HasSuffix(text, ".com...") && !strings.HasSuffix(text, ".org...") { + return false + } + if strings.HasSuffix(text, "…") && !strings.HasSuffix(text, ".com…") && !strings.HasSuffix(text, ".org…") { + return false + } + + // remove any other truncation suffix + text = strings.TrimSuffix(strings.TrimSuffix(text, "..."), "…") + + if len(text) == 0 { + logger.Warn("empty facet text", "text", facet.Text) + return false + } + + // if really not-a-domain, just skip + if !strings.Contains(text, ".") { + return false + } + + // hostnames can't start with a digit (eg, arxiv or DOI links) + for _, c := range text[0:1] { + if unicode.IsNumber(c) { + return false + } + } + + // try to fix any missing method in the text + if !strings.Contains(text, "://") { + text = "https://" + text + } + + // try parsing as a full URL (with whitespace trimmed) + textURL, err := url.Parse(text) + if err != nil { + logger.Warn("invalid link text URL", "url", facet.Text) + return false + } + + // for now just compare domains to handle the most obvious cases + // this public code will obviously get discovered and bypassed. this doesn't earn you any security cred! + linkHost := strings.TrimPrefix(strings.ToLower(linkURL.Host), "www.") + textHost := strings.TrimPrefix(strings.ToLower(textURL.Host), "www.") + if textHost != linkHost { + logger.Warn("misleading mismatched domains", "linkHost", linkURL.Host, "textHost", textURL.Host, "text", facet.Text) + return true + } + return false +} + +var _ automod.PostRuleFunc = MisleadingURLPostRule + +func MisleadingURLPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + // TODO: make this an InSet() config? + if c.Account.Identity.Handle == "nowbreezing.ntw.app" { + return nil + } + facets, err := helpers.ExtractFacets(post) + if err != nil { + c.Logger.Warn("invalid facets", "err", err) + // TODO: or some other "this record is corrupt" indicator? + //c.AddRecordFlag("broken-post") + return nil + } + for _, facet := range facets { + if facet.URL != nil { + if isMisleadingURLFacet(facet, c.Logger) { + c.AddRecordFlag("misleading-link") + c.Notify("slack") + } + } + } + return nil +} + +var _ automod.PostRuleFunc = MisleadingMentionPostRule + +func MisleadingMentionPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + facets, err := helpers.ExtractFacets(post) + if err != nil { + c.Logger.Warn("invalid facets", "err", err) + // TODO: or some other "this record is corrupt" indicator? + //c.AddRecordFlag("broken-post") + return nil + } + for _, facet := range facets { + if facet.DID != nil { + txt := facet.Text + if txt[0] == '@' { + txt = txt[1:] + } + handle, err := syntax.ParseHandle(strings.ToLower(txt)) + if err != nil { + c.Logger.Warn("mention was not a valid handle", "text", txt) + continue + } + + mentioned, err := c.Directory().LookupHandle(c.Ctx, handle) + if err != nil { + c.Logger.Warn("could not resolve handle", "handle", handle) + c.AddRecordFlag("broken-mention") + c.Notify("slack") + break + } + + // TODO: check if mentioned DID was recently updated? might be a caching issue + if mentioned.DID.String() != *facet.DID { + c.Logger.Warn("misleading mention", "text", txt, "did", facet.DID) + c.AddRecordFlag("misleading-mention") + c.Notify("slack") + continue + } + } + } + return nil +} diff --git a/automod/rules/misleading_test.go b/automod/rules/misleading_test.go new file mode 100644 index 000000000..2e47883a6 --- /dev/null +++ b/automod/rules/misleading_test.go @@ -0,0 +1,193 @@ +package rules + +import ( + "bytes" + "context" + "log/slog" + "testing" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/engine" + "github.com/bluesky-social/indigo/automod/helpers" + + "github.com/stretchr/testify/assert" +) + +func TestMisleadingURLPostRule(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + eng := engine.EngineTestFixture() + am1 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + } + cid1 := syntax.CID("cid123") + p1 := appbsky.FeedPost{ + Text: "https://safe.com/ is very reputable", + Facets: []*appbsky.RichtextFacet{ + &appbsky.RichtextFacet{ + Features: []*appbsky.RichtextFacet_Features_Elem{ + &appbsky.RichtextFacet_Features_Elem{ + RichtextFacet_Link: &appbsky.RichtextFacet_Link{ + Uri: "https://evil.com", + }, + }, + }, + Index: &appbsky.RichtextFacet_ByteSlice{ + ByteStart: 0, + ByteEnd: 16, + }, + }, + }, + } + p1buf := new(bytes.Buffer) + assert.NoError(p1.MarshalCBOR(p1buf)) + p1cbor := p1buf.Bytes() + op := engine.RecordOp{ + Action: engine.CreateOp, + DID: am1.Identity.DID, + Collection: syntax.NSID("app.bsky.feed.post"), + RecordKey: syntax.RecordKey("abc123"), + CID: &cid1, + RecordCBOR: p1cbor, + } + c1 := engine.NewRecordContext(ctx, &eng, am1, op) + assert.NoError(MisleadingURLPostRule(&c1, &p1)) + eff1 := engine.ExtractEffects(&c1.BaseContext) + assert.NotEmpty(eff1.RecordFlags) +} + +func TestMisleadingMentionPostRule(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + eng := engine.EngineTestFixture() + am1 := automod.AccountMeta{ + Identity: &identity.Identity{ + DID: syntax.DID("did:plc:abc111"), + Handle: syntax.Handle("handle.example.com"), + }, + } + cid1 := syntax.CID("cid123") + p1 := appbsky.FeedPost{ + Text: "@handle.example.com is a friend", + Facets: []*appbsky.RichtextFacet{ + &appbsky.RichtextFacet{ + Features: []*appbsky.RichtextFacet_Features_Elem{ + &appbsky.RichtextFacet_Features_Elem{ + RichtextFacet_Mention: &appbsky.RichtextFacet_Mention{ + Did: "did:plc:abc222", + }, + }, + }, + Index: &appbsky.RichtextFacet_ByteSlice{ + ByteStart: 1, + ByteEnd: 19, + }, + }, + }, + } + p1buf := new(bytes.Buffer) + assert.NoError(p1.MarshalCBOR(p1buf)) + p1cbor := p1buf.Bytes() + op := engine.RecordOp{ + Action: engine.CreateOp, + DID: am1.Identity.DID, + Collection: syntax.NSID("app.bsky.feed.post"), + RecordKey: syntax.RecordKey("abc123"), + CID: &cid1, + RecordCBOR: p1cbor, + } + c1 := engine.NewRecordContext(ctx, &eng, am1, op) + assert.NoError(MisleadingMentionPostRule(&c1, &p1)) + eff1 := engine.ExtractEffects(&c1.BaseContext) + assert.NotEmpty(eff1.RecordFlags) +} + +func pstr(raw string) *string { + return &raw +} + +func TestIsMisleadingURL(t *testing.T) { + assert := assert.New(t) + logger := slog.Default() + + fixtures := []struct { + facet helpers.PostFacet + out bool + }{ + { + facet: helpers.PostFacet{ + Text: "https://atproto.com", + URL: pstr("https://atproto.com"), + }, + out: false, + }, + { + facet: helpers.PostFacet{ + Text: "https://atproto.com", + URL: pstr("https://evil.com"), + }, + out: true, + }, + { + facet: helpers.PostFacet{ + Text: "https://www.atproto.com", + URL: pstr("https://atproto.com"), + }, + out: false, + }, + { + facet: helpers.PostFacet{ + Text: "https://atproto.com", + URL: pstr("https://www.atproto.com"), + }, + out: false, + }, + { + facet: helpers.PostFacet{ + Text: "[example.com]", + URL: pstr("https://www.example.com"), + }, + out: false, + }, + { + facet: helpers.PostFacet{ + Text: "example.com...", + URL: pstr("https://example.com.evil.com"), + }, + out: true, + }, + { + facet: helpers.PostFacet{ + Text: "ATPROTO.com...", + URL: pstr("https://atproto.com"), + }, + out: false, + }, + { + facet: helpers.PostFacet{ + Text: "1234.5678", + URL: pstr("https://arxiv.org/abs/1234.5678"), + }, + out: false, + }, + { + facet: helpers.PostFacet{ + Text: "www.techdirt.com…", + URL: pstr("https://www.techdirt.com/"), + }, + out: false, + }, + } + + for _, fix := range fixtures { + assert.Equal(fix.out, isMisleadingURLFacet(fix.facet, logger)) + } +} diff --git a/automod/rules/misleading_unicode_reverse.go b/automod/rules/misleading_unicode_reverse.go new file mode 100644 index 000000000..dc75b886a --- /dev/null +++ b/automod/rules/misleading_unicode_reverse.go @@ -0,0 +1,18 @@ +package rules + +import ( + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" +) + +func MisleadingLinkUnicodeReversalPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + + if !strings.Contains(post.Text, "\u202E") { + return nil + } + + c.AddRecordFlag("clickjack-unicode-reversed") + return nil +} diff --git a/automod/rules/nostr.go b/automod/rules/nostr.go new file mode 100644 index 000000000..6ad623ae3 --- /dev/null +++ b/automod/rules/nostr.go @@ -0,0 +1,43 @@ +package rules + +import ( + "strings" + "time" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" +) + +var _ automod.PostRuleFunc = NostrSpamPostRule + +// looks for new accounts, which frequently post the same type of content +func NostrSpamPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { + return nil + } + + // is this a bridged nostr account? if not, bail out + hdl := c.Account.Identity.Handle.String() + if !(strings.HasPrefix(hdl, "npub") && len(hdl) > 63 && strings.HasSuffix(hdl, ".brid.gy")) { + return nil + } + + c.AddAccountFlag("nostr") + + // only posts with dumb patterns (for now) + txt := strings.ToLower(post.Text) + if !c.InSet("trivial-spam-text", txt) { + return nil + } + + // only accounts with empty profile (for now) + if c.Account.Profile.HasAvatar { + return nil + } + + c.ReportAccount(automod.ReportReasonOther, "likely nostr spam account (also labeled; remove label if this isn't spam!)") + c.AddAccountLabel("!hide") + c.Notify("slack") + return nil +} diff --git a/automod/rules/private.go b/automod/rules/private.go new file mode 100644 index 000000000..a897dbe0d --- /dev/null +++ b/automod/rules/private.go @@ -0,0 +1,20 @@ +package rules + +import ( + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" +) + +var _ automod.PostRuleFunc = AccountPrivateDemoPostRule + +// dummy rule. this leaks PII (account email) in logs and should never be used in real life +func AccountPrivateDemoPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if c.Account.Private != nil { + if strings.HasSuffix(c.Account.Private.Email, "@blueskyweb.xyz") { + c.Logger.Info("hello dev!", "email", c.Account.Private.Email) + } + } + return nil +} diff --git a/automod/rules/profile.go b/automod/rules/profile.go new file mode 100644 index 000000000..0ad674d71 --- /dev/null +++ b/automod/rules/profile.go @@ -0,0 +1,42 @@ +package rules + +import ( + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/keyword" +) + +var _ automod.PostRuleFunc = AccountDemoPostRule + +// this is a dummy rule to demonstrate accessing account metadata (eg, profile) from within post handler +func AccountDemoPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if c.Account.Profile.Description != nil && len(post.Text) > 5 && *c.Account.Profile.Description == post.Text { + c.AddRecordFlag("own-profile-description") + c.Notify("slack") + } + return nil +} + +func CelebSpamProfileRule(c *automod.RecordContext, profile *appbsky.ActorProfile) error { + anyElon := false + anyMusk := false + if profile.DisplayName != nil { + tokens := keyword.TokenizeText(*profile.DisplayName) + for _, tok := range tokens { + if tok == "elon" { + anyElon = true + } + if tok == "musk" { + anyMusk = true + } + } + } + if anyElon && anyMusk { + c.AddRecordFlag("profile-elon-musk") + c.ReportAccount(automod.ReportReasonSpam, "possible Elon Musk impersonator") + return nil + } + return nil +} + +var _ automod.ProfileRuleFunc = CelebSpamProfileRule diff --git a/automod/rules/promo.go b/automod/rules/promo.go new file mode 100644 index 000000000..001dbc715 --- /dev/null +++ b/automod/rules/promo.go @@ -0,0 +1,61 @@ +package rules + +import ( + "net/url" + "strings" + "time" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" +) + +var _ automod.PostRuleFunc = AggressivePromotionRule + +// looks for new accounts, with a commercial or donation link in profile, which directly reply to several accounts +// +// this rule depends on ReplyCountPostRule() to set counts +func AggressivePromotionRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 7*24*time.Hour) { + return nil + } + if post.Reply == nil || helpers.IsSelfThread(c, post) { + return nil + } + + allURLs := helpers.ExtractTextURLs(post.Text) + if c.Account.Profile.Description != nil { + profileURLs := helpers.ExtractTextURLs(*c.Account.Profile.Description) + allURLs = append(allURLs, profileURLs...) + } + hasPromo := false + for _, s := range allURLs { + if !strings.Contains(s, "://") { + s = "https://" + s + } + u, err := url.Parse(s) + if err != nil { + c.Logger.Warn("failed to parse URL", "url", s) + continue + } + host := strings.TrimPrefix(strings.ToLower(u.Host), "www.") + if c.InSet("promo-domain", host) { + hasPromo = true + break + } + } + if !hasPromo { + return nil + } + + did := c.Account.Identity.DID.String() + uniqueReplies := c.GetCountDistinct("reply-to", did, countstore.PeriodDay) + if uniqueReplies >= 10 { + c.AddAccountFlag("promo-multi-reply") + c.ReportAccount(automod.ReportReasonSpam, "possible aggressive self-promotion") + c.Notify("slack") + } + + return nil +} diff --git a/automod/rules/quick.go b/automod/rules/quick.go new file mode 100644 index 000000000..127914c7a --- /dev/null +++ b/automod/rules/quick.go @@ -0,0 +1,96 @@ +package rules + +import ( + "fmt" + "strings" + "time" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" +) + +var botLinkStrings = []string{"ainna13762491", "LINK押して", "→ https://tiny", "⇒ http://tiny"} +var botSpamTLDs = []string{".today", ".life"} +var botSpamStrings = []string{"515-9719"} + +var _ automod.ProfileRuleFunc = BotLinkProfileRule + +func BotLinkProfileRule(c *automod.RecordContext, profile *appbsky.ActorProfile) error { + if profile.Description != nil { + for _, str := range botLinkStrings { + if strings.Contains(*profile.Description, str) { + c.AddAccountFlag("profile-bot-string") + c.AddAccountLabel("spam") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("possible bot based on link in profile: %s", str)) + c.Notify("slack") + return nil + } + } + if strings.Contains(*profile.Description, "🏈🍕🌀") { + c.AddAccountFlag("profile-bot-string") + c.ReportAccount(automod.ReportReasonSpam, "possible bot based on string in profile") + c.Notify("slack") + return nil + } + } + return nil +} + +var _ automod.PostRuleFunc = SimpleBotPostRule + +func SimpleBotPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + for _, str := range botSpamStrings { + if strings.Contains(post.Text, str) { + // NOTE: reporting the *account* not individual posts + c.AddAccountFlag("post-bot-string") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("possible bot based on string in post: %s", str)) + c.Notify("slack") + return nil + } + } + return nil +} + +var _ automod.IdentityRuleFunc = NewAccountBotEmailRule + +func NewAccountBotEmailRule(c *automod.AccountContext) error { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(c, 1*time.Hour) { + return nil + } + + for _, tld := range botSpamTLDs { + if strings.HasSuffix(c.Account.Private.Email, tld) { + c.AddAccountFlag("new-suspicious-email") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("possible bot based on email domain TLD: %s", tld)) + c.Notify("slack") + return nil + } + } + return nil +} + +var _ automod.PostRuleFunc = TrivialSpamPostRule + +// looks for new accounts, which frequently post the same type of content +func TrivialSpamPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 8*24*time.Hour) { + return nil + } + + // only posts with dumb patterns (for now) + txt := strings.ToLower(post.Text) + if !c.InSet("trivial-spam-text", txt) { + return nil + } + + // only accounts with empty profile (for now) + if c.Account.Profile.HasAvatar { + return nil + } + + c.ReportAccount(automod.ReportReasonOther, "trivial spam account (also labeled; remove label if this isn't spam!)") + c.AddAccountLabel("!hide") + c.Notify("slack") + return nil +} diff --git a/automod/rules/replies.go b/automod/rules/replies.go new file mode 100644 index 000000000..e03e9de53 --- /dev/null +++ b/automod/rules/replies.go @@ -0,0 +1,170 @@ +package rules + +import ( + "fmt" + "time" + "unicode/utf8" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" +) + +var _ automod.PostRuleFunc = ReplyCountPostRule + +// does not count "self-replies" (direct to self, or in own post thread) +func ReplyCountPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if post.Reply == nil || helpers.IsSelfThread(c, post) { + return nil + } + + did := c.Account.Identity.DID.String() + if c.GetCount("reply", did, countstore.PeriodDay) > 3 { + // TODO: disabled, too noisy for prod + //c.AddAccountFlag("frequent-replier") + } + c.Increment("reply", did) + + parentURI, err := syntax.ParseATURI(post.Reply.Parent.Uri) + if err != nil { + c.Logger.Warn("failed to parse reply AT-URI", "uri", post.Reply.Parent.Uri) + return nil + } + c.IncrementDistinct("reply-to", did, parentURI.Authority().String()) + return nil +} + +// triggers on the N+1 post +// var identicalReplyLimit = 6 +// TODO: bumping temporarily +var identicalReplyLimit = 20 +var identicalReplyActionLimit = 75 + +var _ automod.PostRuleFunc = IdenticalReplyPostRule + +// Looks for accounts posting the exact same text multiple times. Does not currently count the number of distinct accounts replied to, just counts replies at all. +// +// There can be legitimate situations that trigger this rule, so in most situations should be a "report" not "label" action. +func IdenticalReplyPostRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if post.Reply == nil || helpers.IsSelfThread(c, post) { + return nil + } + + // don't action short replies, or accounts more than two weeks old + if utf8.RuneCountInString(post.Text) <= 10 { + return nil + } + if helpers.AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { + return nil + } + + // don't count if there is a follow-back relationship + if helpers.ParentOrRootIsFollower(c, post) { + return nil + } + + // increment before read. use a specific period (IncrementPeriod()) to reduce the number of counters (one per unique post text) + period := countstore.PeriodDay + bucket := c.Account.Identity.DID.String() + "/" + helpers.HashOfString(post.Text) + c.IncrementPeriod("reply-text", bucket, period) + + count := c.GetCount("reply-text", bucket, period) + if count >= identicalReplyLimit { + c.AddAccountFlag("multi-identical-reply") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("possible spam (new account, %d identical reply-posts today)", count)) + c.Notify("slack") + } + if count >= identicalReplyActionLimit && utf8.RuneCountInString(post.Text) > 100 { + c.ReportAccount(automod.ReportReasonRude, fmt.Sprintf("likely spam/harassment (new account, %d identical reply-posts today), actioned (remove label urgently if account is ok)", count)) + c.AddAccountLabel("!warn") + c.Notify("slack") + } + + return nil +} + +// Similar to above rule but only counts replies to the same post. More aggressively applies a spam label to new accounts that are less than a day old. +var identicalReplySameParentLimit = 3 +var identicalReplySameParentMaxAge = 24 * time.Hour +var identicalReplySameParentMaxPosts int64 = 50 +var _ automod.PostRuleFunc = IdenticalReplyPostSameParentRule + +func IdenticalReplyPostSameParentRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + if post.Reply == nil || helpers.IsSelfThread(c, post) { + return nil + } + + if helpers.ParentOrRootIsFollower(c, post) { + return nil + } + + postCount := c.Account.PostsCount + if helpers.AccountIsOlderThan(&c.AccountContext, identicalReplySameParentMaxAge) || postCount >= identicalReplySameParentMaxPosts { + return nil + } + + period := countstore.PeriodHour + bucket := c.Account.Identity.DID.String() + "/" + post.Reply.Parent.Uri + "/" + helpers.HashOfString(post.Text) + c.IncrementPeriod("reply-text-same-post", bucket, period) + + count := c.GetCount("reply-text-same-post", bucket, period) + if count >= identicalReplySameParentLimit { + c.AddAccountFlag("multi-identical-reply-same-post") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("possible spam (%d identical reply-posts to same post today)", count)) + c.AddAccountLabel("spam") + c.Notify("slack") + } + + return nil +} + +// TODO: bumping temporarily +// var youngReplyAccountLimit = 12 +var youngReplyAccountLimit = 200 +var _ automod.PostRuleFunc = YoungAccountDistinctRepliesRule + +func YoungAccountDistinctRepliesRule(c *automod.RecordContext, post *appbsky.FeedPost) error { + // only replies, and skip self-replies (eg, threads) + if post.Reply == nil || helpers.IsSelfThread(c, post) { + return nil + } + + // don't action short replies, or accounts more than two weeks old + if utf8.RuneCountInString(post.Text) <= 10 { + return nil + } + if helpers.AccountIsOlderThan(&c.AccountContext, 14*24*time.Hour) { + return nil + } + + // don't count if there is a follow-back relationship + if helpers.ParentOrRootIsFollower(c, post) { + return nil + } + + parentURI, err := syntax.ParseATURI(post.Reply.Parent.Uri) + if err != nil { + c.Logger.Warn("failed to parse reply AT-URI", "uri", post.Reply.Parent.Uri) + return nil + } + parentDID, err := parentURI.Authority().AsDID() + if err != nil { + c.Logger.Warn("reply AT-URI authority not a DID", "uri", post.Reply.Parent.Uri) + return nil + } + + did := c.Account.Identity.DID.String() + + c.IncrementDistinct("young-reply-to", did, parentDID.String()) + // NOTE: won't include the increment from this event + count := c.GetCountDistinct("young-reply-to", did, countstore.PeriodHour) + if count >= youngReplyAccountLimit { + c.AddAccountFlag("new-account-distinct-account-reply") + c.ReportAccount(automod.ReportReasonRude, fmt.Sprintf("possible spam (new account, reply-posts to %d distinct accounts in past hour)", count)) + c.Notify("slack") + } + + return nil +} diff --git a/automod/rules/replies_test.go b/automod/rules/replies_test.go new file mode 100644 index 000000000..9f86f0718 --- /dev/null +++ b/automod/rules/replies_test.go @@ -0,0 +1,33 @@ +package rules + +import ( + "context" + "testing" + + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/capture" + "github.com/bluesky-social/indigo/automod/engine" + + "github.com/stretchr/testify/assert" +) + +func TestIdenticalReplyPostRule(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + + eng := engine.EngineTestFixture() + eng.Rules = automod.RuleSet{ + PostRules: []automod.PostRuleFunc{ + IdenticalReplyPostRule, + }, + } + + cap := capture.MustLoadCapture("testdata/capture_hackerdarkweb.json") + did := cap.AccountMeta.Identity.DID.String() + assert.NoError(capture.ProcessCaptureRules(&eng, cap)) + f, err := eng.Flags.Get(ctx, did) + assert.NoError(err) + // TODO: tweaked threshold, disabling for now + _ = f + //assert.Equal([]string{"multi-identical-reply"}, f) +} diff --git a/automod/rules/reposts.go b/automod/rules/reposts.go new file mode 100644 index 000000000..573146558 --- /dev/null +++ b/automod/rules/reposts.go @@ -0,0 +1,51 @@ +package rules + +import ( + "fmt" + "strings" + "time" + + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/helpers" +) + +var dailyRepostThresholdWithoutPost = 30 +var dailyRepostThresholdWithLowPost = 100 +var dailyPostThresholdWithHighRepost = 5 + +var _ automod.RecordRuleFunc = TooManyRepostRule + +// looks for accounts which do frequent reposts +func TooManyRepostRule(c *automod.RecordContext) error { + // Don't bother checking reposts from accounts older than 30 days + if c.Account.Identity == nil || !helpers.AccountIsYoungerThan(&c.AccountContext, 30*24*time.Hour) { + return nil + } + + did := c.Account.Identity.DID.String() + + // Special case for newsmast bridge feeds + handle := c.Account.Identity.Handle.String() + if strings.HasSuffix(handle, ".ap.brid.gy") { + return nil + } + + switch c.RecordOp.Collection { + case "app.bsky.feed.post": + c.Increment("post", did) + case "app.bsky.feed.repost": + c.Increment("repost", did) + // +1 to avoid potential divide by 0 issue + repostCount := c.GetCount("repost", did, countstore.PeriodDay) + postCount := c.GetCount("post", did, countstore.PeriodDay) + highRepost := (repostCount >= dailyRepostThresholdWithoutPost && postCount < 1) || (repostCount >= dailyRepostThresholdWithLowPost && postCount < dailyPostThresholdWithHighRepost) + if highRepost { + c.Logger.Info("high-repost-count", "reposted-today", repostCount, "posted-today", postCount) + c.AddAccountFlag("high-repost-count") + c.ReportAccount(automod.ReportReasonSpam, fmt.Sprintf("too many reposts: %d reposts, %d posts today (so far)", repostCount, postCount)) + c.Notify("slack") + } + } + return nil +} diff --git a/automod/rules/testdata/capture_hackerdarkweb.json b/automod/rules/testdata/capture_hackerdarkweb.json new file mode 100644 index 000000000..7cdb2f4a3 --- /dev/null +++ b/automod/rules/testdata/capture_hackerdarkweb.json @@ -0,0 +1,521 @@ +{ + "capturedAt": "2023-12-09T05:50:42.004Z", + "accountMeta": { + "Identity": { + "DID": "did:plc:suvj7i7gk6vzc7wvwd3gx6zq", + "Handle": "hackerdarkweb.bsky.social", + "AlsoKnownAs": [ + "at://hackerdarkweb.bsky.social" + ], + "Services": { + "atproto_pds": { + "Type": "AtprotoPersonalDataServer", + "URL": "https://puffball.us-east.host.bsky.network" + } + }, + "Keys": { + "atproto": { + "Type": "Multikey", + "PublicKeyMultibase": "zQ3sha9ULLgA6zyJmk1z2uDkpkSs4Ypou33RE8XuhwWtSHF75" + } + } + }, + "Profile": { + "HasAvatar": true, + "Description": "Message on WhatsApp to get cyber help support \n‪+1 (765) 734‑3839‬\nPRO HACKER🔉\nCAR TRACKING \nBINARY HACKING🔉\nACCOUNT RECOVERY💭\nSPY ON PARTNERS PHONE💭\nRECOVER SCAMMED FUNDS\nUNLOCKING AND TRACKING DEVICES\nBOT INVESTMENT", + "DisplayName": "Hacker_dark_web" + }, + "Private": null, + "AccountLabels": [ + "spam" + ], + "AccountNegatedLabels": null, + "AccountFlags": [], + "FollowersCount": 3, + "FollowsCount": 102, + "PostsCount": 140, + "Takendown": false + }, + "postRecords": [ + { + "cid": "bafyreietl6p56eogohp2cnm6k6isod7jtkbh7k3oxrrspjv7uvlwlorgyy", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxt2dprhx22", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:49:34.125Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:zhxe4qbuwbmvliuh7subna6p" + } + ], + "index": { + "byteEnd": 24, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@crossbunbun.bsky.social Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreibnj3dh27zcuywx6bcbgde74f57sqjsly2iirkam2kwbztckiuzu4", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxt24db3v2f", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:49:26.393Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreiai3o2b7d5alheiiohqb3zeagplz37lngvndowzabzy2ox2gil3ji", + "uri": "at://did:plc:zhxe4qbuwbmvliuh7subna6p/app.bsky.feed.post/3kfrqvnrbjk22" + }, + "root": { + "cid": "bafyreiai3o2b7d5alheiiohqb3zeagplz37lngvndowzabzy2ox2gil3ji", + "uri": "at://did:plc:zhxe4qbuwbmvliuh7subna6p/app.bsky.feed.post/3kfrqvnrbjk22" + } + }, + "text": "Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreihtphceqkvhjfogyiwebyytddthogzdsysigopupja6wefdfdkvrm", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxszcwuyl23", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:48:59.765Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:jhp4iwpyeqtcsrp6kifcgi7b" + } + ], + "index": { + "byteEnd": 25, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@cyberfoxmeow.bsky.social Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreidwryv6aofdrla243zelnspet4o66eet2pmd2vlla3yrfjeqlju2u", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxsz257d322", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:48:50.538Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreiguf5uuetq5fvr7bgh4vt42q6wzf34wlwseq5jz2s6elwuxbjtaym", + "uri": "at://did:plc:jhp4iwpyeqtcsrp6kifcgi7b/app.bsky.feed.post/3kftvcwhm4m2g" + }, + "root": { + "cid": "bafyreiguf5uuetq5fvr7bgh4vt42q6wzf34wlwseq5jz2s6elwuxbjtaym", + "uri": "at://did:plc:jhp4iwpyeqtcsrp6kifcgi7b/app.bsky.feed.post/3kftvcwhm4m2g" + } + }, + "text": "Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreiawlfmqoxkjm4woqc36nl4armlmyo26shkfgah7asne2baka43uhy", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxsyobb4s2z", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:48:38.093Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:bzy5rjjduvvkxno5xe3evl3f" + } + ], + "index": { + "byteEnd": 24, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@lollardfish.bsky.social Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreif3uvcp7djvt6nyqcxoh66coigdjbvtchsgybmrcj73wemaqalr4a", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxsygqc6v2a", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:48:30.192Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreihwjky3uxjzr5bzdty2d6acbdcjo6zoloxlqhbg3swo4v74struf4", + "uri": "at://did:plc:bzy5rjjduvvkxno5xe3evl3f/app.bsky.feed.post/3kfvfhdu5xi2p" + }, + "root": { + "cid": "bafyreihwjky3uxjzr5bzdty2d6acbdcjo6zoloxlqhbg3swo4v74struf4", + "uri": "at://did:plc:bzy5rjjduvvkxno5xe3evl3f/app.bsky.feed.post/3kfvfhdu5xi2p" + } + }, + "text": "Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreign2datfgnkwajm3zffpbmsyczraifte3ui3p4mzfwmc5hamd4xqq", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxsx5p6oy2s", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:47:47.163Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreifom67hymr6xtgzn73didgft27wo5w2kky2g5zktjrpiakqelksee", + "uri": "at://did:plc:c2pocrpcqo5oqts2udoapepb/app.bsky.feed.post/3kfxoxp67yn26" + }, + "root": { + "cid": "bafyreifom67hymr6xtgzn73didgft27wo5w2kky2g5zktjrpiakqelksee", + "uri": "at://did:plc:c2pocrpcqo5oqts2udoapepb/app.bsky.feed.post/3kfxoxp67yn26" + } + }, + "text": "Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreifxevvtccaebv34dnemdb2bvyfj34jcakfqh6eptgaaaxmm4vdtja", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxswjdi5t23", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:47:25.788Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:c2pocrpcqo5oqts2udoapepb" + } + ], + "index": { + "byteEnd": 20, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@johncnj.bsky.social Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreiauo4dhxpjycqqn7gmv276224x3shnc2paf44eonbatqptk5b74ti", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfxswalsxn2j", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-07T16:47:16.632Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreibqcgtk44ovhreovuvp5f3z6jfwhlahktysizlgmvs5i6eajrfnby", + "uri": "at://did:plc:c2pocrpcqo5oqts2udoapepb/app.bsky.feed.post/3kfxlfqfzll2p" + }, + "root": { + "cid": "bafyreibqcgtk44ovhreovuvp5f3z6jfwhlahktysizlgmvs5i6eajrfnby", + "uri": "at://did:plc:c2pocrpcqo5oqts2udoapepb/app.bsky.feed.post/3kfxlfqfzll2p" + } + }, + "text": "Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreieapui5bwfereerup47apqbqhfzkp3eussbff3psfq5pkzpjz5xqy", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfprrztvpk2s", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T12:05:43.694Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:xuqnzpixtod7hf7fqe4dvepg" + } + ], + "index": { + "byteEnd": 18, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@dwuff.bsky.social Message on WhatsApp to recover your lost funds \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreibe5tvxgexdvtzhjywgdrdmehy566g5xn33vdxihf6pff5ztne6oi", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfprrjkzev2i", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T12:05:26.627Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreifretb4p2o4u22j3kaeckh5z2cs2xseqr3tqjfkae52zlf5omzwzq", + "uri": "at://did:plc:xuqnzpixtod7hf7fqe4dvepg/app.bsky.feed.post/3kfnoc26bgk2f" + }, + "root": { + "cid": "bafyreifretb4p2o4u22j3kaeckh5z2cs2xseqr3tqjfkae52zlf5omzwzq", + "uri": "at://did:plc:xuqnzpixtod7hf7fqe4dvepg/app.bsky.feed.post/3kfnoc26bgk2f" + } + }, + "text": "Message on WhatsApp to recover your lost funds \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreichajoc5lc2uhynxhnosmvp2zjgg5iw3tlqrrrwmgxidl5ovdpyaq", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfprq5hine2a", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T12:04:40.380Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreiavkwoo52ye6ltwgl4yk5hepsyykviaihzkfn4hapujnipd24ejua", + "uri": "at://did:plc:4rfp3cq3ycsfg6owkqrex5ls/app.bsky.feed.post/3kfnwjhxtpf22" + }, + "root": { + "cid": "bafyreib34kaul7stegzmkczlfxgpr65rn5kxvjwexk6n6cuiluw4i447ii", + "uri": "at://did:plc:4rfp3cq3ycsfg6owkqrex5ls/app.bsky.feed.post/3kfnrapfsyu2j" + } + }, + "text": "Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreidjkhdtcbyjhnpcg5vgditswek4pa3vvnbw3nii2vxuibwf2n4p6i", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfprpx6zw32m", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T12:04:33.781Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:4rfp3cq3ycsfg6owkqrex5ls" + } + ], + "index": { + "byteEnd": 25, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@carrie4jesus.bsky.social Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreid723xkolnyvugb6lwflatnz7dmegswmk57vsbwwf2mutnayyadve", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfprmof5k42o", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T12:02:43.894Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:x7bzd43ztwpxpaxgsi7nxx7v" + } + ], + "index": { + "byteEnd": 19, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@ehhjax.bsky.social Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreibcwutbkbfdwjcwsur2si4gjhf3smut64lejllcjmw5cheg7y37iu", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfprm7vkha2g", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T12:02:28.704Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:kgpjze4ebmgd2mwa5aqrijjc" + } + ], + "index": { + "byteEnd": 27, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@sunshinesdaily.bsky.social Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreic6oplgskdo3jix7pimcvlpf5ihc576gwdkwsfi7by33tn4z5icgu", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kfprlvcphs2k", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-12-04T12:02:17.612Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreicillzq2egsxq5e653s4mhl7z6mcoggeriyq27pjhirzn57de7ogy", + "uri": "at://did:plc:kgpjze4ebmgd2mwa5aqrijjc/app.bsky.feed.post/3kfojibjwpy2p" + }, + "root": { + "cid": "bafyreia5rscvtshpgfoeczgfv7kgior62louxlmrnsua6j562iiu65sj24", + "uri": "at://did:plc:x7bzd43ztwpxpaxgsi7nxx7v/app.bsky.feed.post/3kfoifgbwyr27" + } + }, + "text": "Message on WhatsApp to recover your lost account \n\n‪+1 (765) 734‑3839‬" + } + }, + { + "cid": "bafyreicp4fq6py6udl3qsen3ug5r6xwbptetahyz33cwhi3g74lxoqo7ga", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kf7gku22an2a", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-28T00:02:15.718Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:2jnktzctncait2dgiro7z27j" + } + ], + "index": { + "byteEnd": 23, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@zaffreyael.bsky.social PRO HACKER\nMESSAGE ON WHATSAPP +1 (412) 568-3582\nFor the following services send me a message \n1.Cloning your partner WhatsApp to see who your partner is talking to on Whatsapp\n2.Recovery of all social media accounts Facebook, Instagram,YouTube Pinterest Twitter etc" + } + }, + { + "cid": "bafyreiex4zxkelnq4bxhzz7zwpzolpw3jsxke2kw4y2uhs6peahkkok4bm", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kf7gkh27r22s", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-28T00:02:01.996Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreiefwnjmxf6bx7effff5ypjah3nx7e7hfr3jksppllihqirhmtpblq", + "uri": "at://did:plc:2jnktzctncait2dgiro7z27j/app.bsky.feed.post/3kex2nuj4vs2j" + }, + "root": { + "cid": "bafyreiefwnjmxf6bx7effff5ypjah3nx7e7hfr3jksppllihqirhmtpblq", + "uri": "at://did:plc:2jnktzctncait2dgiro7z27j/app.bsky.feed.post/3kex2nuj4vs2j" + } + }, + "text": "PRO HACKER\nMESSAGE ON WHATSAPP +1 (412) 568-3582\nFor the following services send me a message \n1.Cloning your partner WhatsApp to see who your partner is talking to on Whatsapp\n2.Recovery of all social media accounts Facebook, Instagram,YouTube Pinterest Twitter etc\n3.Unlocking of iCloud on iPhones" + } + }, + { + "cid": "bafyreiagl6qpgtpxbzwezoa5c2kkzxrqf5n4hvo7wplmx7upvbcq5htez4", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kf7gjqujsc2s", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-28T00:01:38.855Z", + "facets": [ + { + "features": [ + { + "$type": "app.bsky.richtext.facet#mention", + "did": "did:plc:axxayzoylwiyczff6nso7z2y" + } + ], + "index": { + "byteEnd": 26, + "byteStart": 0 + } + } + ], + "langs": [ + "en" + ], + "text": "@ashesinadream.bsky.social PRO HACKER\nMESSAGE ON WHATSAPP +1 (412) 568-3582\nFor the following services send me a message \n1.Cloning your partner WhatsApp to see who your partner is talking to on Whatsapp\n2.Recovery of all social media accounts Facebook, Instagram,YouTube Pinterest Twitter etc" + } + }, + { + "cid": "bafyreidxqyocxlwrfbftuugzg7mj2wd5i6twnwmbu7refb6cf4jjwe7zne", + "uri": "at://did:plc:suvj7i7gk6vzc7wvwd3gx6zq/app.bsky.feed.post/3kf7givmebs2j", + "value": { + "$type": "app.bsky.feed.post", + "createdAt": "2023-11-28T00:01:10.283Z", + "langs": [ + "en" + ], + "reply": { + "parent": { + "cid": "bafyreiawxmoft5bjla57qoccflkiir7k7ztyegdfoi5od2npzmvq7gw5iu", + "uri": "at://did:plc:axxayzoylwiyczff6nso7z2y/app.bsky.feed.post/3kexgryq5d62a" + }, + "root": { + "cid": "bafyreiawxmoft5bjla57qoccflkiir7k7ztyegdfoi5od2npzmvq7gw5iu", + "uri": "at://did:plc:axxayzoylwiyczff6nso7z2y/app.bsky.feed.post/3kexgryq5d62a" + } + }, + "text": "PRO HACKER\nMESSAGE ON WHATSAPP +1 (412) 568-3582\nFor the following services send me a message \n1.Cloning your partner WhatsApp to see who your partner is talking to on Whatsapp\n2.Recovery of all social media accounts Facebook, Instagram,YouTube Pinterest Twitter etc\n3.Unlocking of iCloud on iPhones" + } + } + ] +} diff --git a/automod/setstore/doc.go b/automod/setstore/doc.go new file mode 100644 index 000000000..df1c0067b --- /dev/null +++ b/automod/setstore/doc.go @@ -0,0 +1,2 @@ +// Interface for simple sets of strings, with fast inclusion checks. +package setstore diff --git a/automod/setstore/setstore.go b/automod/setstore/setstore.go new file mode 100644 index 000000000..0f5bedb5c --- /dev/null +++ b/automod/setstore/setstore.go @@ -0,0 +1,61 @@ +package setstore + +import ( + "context" + "encoding/json" + "io" + "os" +) + +type SetStore interface { + InSet(ctx context.Context, name, val string) (bool, error) +} + +// TODO: this implementation isn't race-safe (yet)! +type MemSetStore struct { + Sets map[string]map[string]bool +} + +func NewMemSetStore() MemSetStore { + return MemSetStore{ + Sets: make(map[string]map[string]bool), + } +} + +func (s MemSetStore) InSet(ctx context.Context, name, val string) (bool, error) { + set, ok := s.Sets[name] + if !ok { + // NOTE: currently returns false when entire set isn't found + return false, nil + } + _, ok = set[val] + return ok, nil +} + +func (s *MemSetStore) LoadFromFileJSON(p string) error { + + f, err := os.Open(p) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + raw, err := io.ReadAll(f) + if err != nil { + return err + } + + var rules map[string][]string + if err := json.Unmarshal(raw, &rules); err != nil { + return err + } + + for name, l := range rules { + m := make(map[string]bool, len(l)) + for _, val := range l { + m[val] = true + } + s.Sets[name] = m + } + return nil +} diff --git a/automod/visual/abyss_api.go b/automod/visual/abyss_api.go new file mode 100644 index 000000000..665b2e2fd --- /dev/null +++ b/automod/visual/abyss_api.go @@ -0,0 +1,42 @@ +package visual + +import ( + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +type AbyssScanResp struct { + Blob *lexutil.LexBlob `json:"blob"` + Match *AbyssMatchResult `json:"match,omitempty"` + Classify *AbyssClassifyResult `json:"classify,omitempty"` + Review *AbyssReviewState `json:"review,omitempty"` +} + +type AbyssMatchResult struct { + Status string `json:"status"` + Hits []AbyssMatchHit `json:"hits"` +} + +type AbyssMatchHit struct { + HashType string `json:"hashType,omitempty"` + HashValue string `json:"hashValue,omitempty"` + Label string `json:"label,omitempty"` + // TODO: Corpus +} + +type AbyssClassifyResult struct { + // TODO +} + +type AbyssReviewState struct { + State string `json:"state,omitempty"` + TicketID string `json:"ticketId,omitempty"` +} + +func (amr *AbyssMatchResult) IsAbuseMatch() bool { + for _, hit := range amr.Hits { + if hit.Label == "csam" || hit.Label == "csem" { + return true + } + } + return false +} diff --git a/automod/visual/abyss_client.go b/automod/visual/abyss_client.go new file mode 100644 index 000000000..b5252b0d2 --- /dev/null +++ b/automod/visual/abyss_client.go @@ -0,0 +1,89 @@ +package visual + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "time" + + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/util" + + "github.com/earthboundkid/versioninfo/v2" +) + +type AbyssClient struct { + Client http.Client + Host string + Password string + RatelimitBypass string +} + +func NewAbyssClient(host, password, ratelimitBypass string) AbyssClient { + return AbyssClient{ + Client: *util.RobustHTTPClient(), + Host: host, + Password: password, + RatelimitBypass: password, + } +} + +func (ac *AbyssClient) ScanBlob(ctx context.Context, blob lexutil.LexBlob, blobBytes []byte, params map[string]string) (*AbyssScanResp, error) { + + slog.Debug("sending blob to abyss", "cid", blob.Ref.String(), "mimetype", blob.MimeType, "size", len(blobBytes)) + + body := bytes.NewBuffer(blobBytes) + req, err := http.NewRequest("POST", ac.Host+"/xrpc/com.atproto.unspecced.scanBlob", body) + if err != nil { + return nil, err + } + + q := req.URL.Query() + for k, v := range params { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + + req.SetBasicAuth("admin", ac.Password) + req.Header.Add("Content-Type", blob.MimeType) + req.Header.Add("Content-Length", fmt.Sprintf("%d", blob.Size)) + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "indigo-automod/"+versioninfo.Short()) + if ac.RatelimitBypass != "" { + req.Header.Set("x-ratelimit-bypass", ac.RatelimitBypass) + } + + start := time.Now() + defer func() { + duration := time.Since(start) + abyssAPIDuration.Observe(duration.Seconds()) + }() + + req = req.WithContext(ctx) + res, err := ac.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("abyss request failed: %v", err) + } + defer res.Body.Close() + + abyssAPICount.WithLabelValues(fmt.Sprint(res.StatusCode)).Inc() + if res.StatusCode != 200 { + return nil, fmt.Errorf("abyss request failed statusCode=%d", res.StatusCode) + } + + respBytes, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read abyss resp body: %v", err) + } + + var respObj AbyssScanResp + if err := json.Unmarshal(respBytes, &respObj); err != nil { + return nil, fmt.Errorf("failed to parse abyss resp JSON: %v", err) + } + slog.Info("abyss-scan-response", "cid", blob.Ref.String(), "obj", respObj) + return &respObj, nil +} diff --git a/automod/visual/abyss_rule.go b/automod/visual/abyss_rule.go new file mode 100644 index 000000000..116504d6d --- /dev/null +++ b/automod/visual/abyss_rule.go @@ -0,0 +1,47 @@ +package visual + +import ( + "strings" + + "github.com/bluesky-social/indigo/automod" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func (ac *AbyssClient) AbyssScanBlobRule(c *automod.RecordContext, blob lexutil.LexBlob, data []byte) error { + + if !strings.HasPrefix(blob.MimeType, "image/") { + return nil + } + + params := make(map[string]string) + params["did"] = c.Account.Identity.DID.String() + if !c.Account.Identity.Handle.IsInvalidHandle() { + params["handle"] = c.Account.Identity.Handle.String() + } + if c.Account.Private != nil && c.Account.Private.Email != "" { + params["accountEmail"] = c.Account.Private.Email + } + params["uri"] = c.RecordOp.ATURI().String() + + resp, err := ac.ScanBlob(c.Ctx, blob, data, params) + if err != nil { + return err + } + + if resp.Match == nil || resp.Match.Status != "success" { + // TODO: should this return an error, or just log? + c.Logger.Error("abyss blob scan failed", "cid", blob.Ref.String()) + return nil + } + + if resp.Match.IsAbuseMatch() { + c.Logger.Warn("abyss blob match", "cid", blob.Ref.String()) + c.AddRecordFlag("abyss-match") + c.TakedownRecord() + // purge blob as part of record takedown + c.TakedownBlob(blob.Ref.String()) + c.ReportRecord(automod.ReportReasonViolation, "possible CSAM image match; post has been takendown while verifying.\nAccount should be reviewed for any other content") + } + + return nil +} diff --git a/automod/visual/doc.go b/automod/visual/doc.go new file mode 100644 index 000000000..ad30c9992 --- /dev/null +++ b/automod/visual/doc.go @@ -0,0 +1,2 @@ +// automod helpers for visual content (image blobs) +package visual diff --git a/automod/visual/hiveai_client.go b/automod/visual/hiveai_client.go new file mode 100644 index 000000000..2e2f75cf1 --- /dev/null +++ b/automod/visual/hiveai_client.go @@ -0,0 +1,232 @@ +package visual + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "mime/multipart" + "net/http" + "time" + + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/util" + + "github.com/earthboundkid/versioninfo/v2" +) + +type HiveAIClient struct { + Client http.Client + ApiToken string + + PreScreenClient *PreScreenClient +} + +// schema: https://docs.thehive.ai/reference/classification +type HiveAIResp struct { + Status []HiveAIResp_Status `json:"status"` +} + +type HiveAIResp_Status struct { + Response HiveAIResp_Response `json:"response"` +} + +type HiveAIResp_Response struct { + Output []HiveAIResp_Out `json:"output"` +} + +type HiveAIResp_Out struct { + Time float64 `json:"time"` + Classes []HiveAIResp_Class `json:"classes"` +} + +type HiveAIResp_Class struct { + Class string `json:"class"` + Score float64 `json:"score"` +} + +func NewHiveAIClient(token string) HiveAIClient { + return HiveAIClient{ + Client: *util.RobustHTTPClient(), + ApiToken: token, + } +} + +// Simple direct mappings from individual classes to to labels +// +// hive gore and violence: https://docs.thehive.ai/docs/class-descriptions-violence-gore +func summarizeSimpleLabels(cl []HiveAIResp_Class) []string { + var labels []string + + for _, cls := range cl { + if cls.Class == "very_bloody" && cls.Score >= 0.90 { + labels = append(labels, "graphic-media") + } + if cls.Class == "human_corpse" && cls.Score >= 0.90 { + labels = append(labels, "graphic-media") + } + if cls.Class == "hanging" && cls.Score >= 0.90 { + labels = append(labels, "graphic-media") + } + if cls.Class == "yes_self_harm" && cls.Score >= 0.96 { + labels = append(labels, "self-harm") + } + } + return labels +} + +// Matches only one (or none) of: porn, sexual, nudity +// +// porn: sexual and nudity. including both explicit activity or full-frontal and suggestive/intent +// sexual: sexually suggestive, not explicit; may include some forms of nudity +// nudity: non-sexual nudity (eg, artistic, possibly some photographic) +// +// hive docs/definitions: https://docs.thehive.ai/docs/sexual-content +func summarizeSexualLabels(cl []HiveAIResp_Class) string { + + scores := make(map[string]float64) + for _, cls := range cl { + scores[cls.Class] = cls.Score + } + + threshold := 0.90 + + // if this is furry art content, then require very high confidence when flagging for any sexual reason + // note that this is a custom model, not always returned in generic Hive responses + if furryScore, ok := scores["furry-yes_furry"]; ok && furryScore > 0.95 { + threshold = 0.99 + } + + // first check if porn... + for _, pornClass := range []string{"yes_sexual_activity", "animal_genitalia_and_human", "yes_realistic_nsfw"} { + if scores[pornClass] >= threshold { + return "porn" + } + } + if scores["general_nsfw"] >= threshold { + // special case for some anime examples + if scores["animated_animal_genitalia"] >= 0.5 { + return "porn" + } + + // special case for some pornographic/explicit classic drawings + if scores["yes_undressed"] >= threshold && scores["yes_sexual_activity"] >= threshold { + return "porn" + } + } + + // then check for sexual suggestive (which may include nudity)... + for _, sexualClass := range []string{"yes_sexual_intent", "yes_sex_toy"} { + if scores[sexualClass] >= threshold { + return "sexual" + } + } + if scores["yes_undressed"] >= threshold { + // special case for bondage examples + if scores["yes_sex_toy"] > 0.75 { + return "sexual" + } + } + + // then non-sexual nudity... + for _, nudityClass := range []string{"yes_male_nudity", "yes_female_nudity", "yes_undressed"} { + if scores[nudityClass] >= threshold { + return "nudity" + } + } + + // then finally flag remaining "underwear" images in to sexually suggestive + // (after non-sexual content already labeled above) + for _, underwearClass := range []string{"yes_male_underwear", "yes_female_underwear"} { + // TODO: experimenting with higher threshhold during traffic spike + //if scores[underwearClass] >= threshold { + if scores[underwearClass] >= 0.98 { + return "sexual" + } + } + + return "" +} + +func (resp *HiveAIResp) SummarizeLabels() []string { + var labels []string + + for _, status := range resp.Status { + for _, out := range status.Response.Output { + simple := summarizeSimpleLabels(out.Classes) + if len(simple) > 0 { + labels = append(labels, simple...) + } + + sexual := summarizeSexualLabels(out.Classes) + if sexual != "" { + labels = append(labels, sexual) + } + } + } + + return labels +} + +func (hal *HiveAIClient) LabelBlob(ctx context.Context, blob lexutil.LexBlob, blobBytes []byte) ([]string, error) { + + slog.Debug("sending blob to Hive AI", "cid", blob.Ref.String(), "mimetype", blob.MimeType, "size", len(blobBytes)) + + // generic HTTP form file upload, then parse the response JSON + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("media", blob.Ref.String()) + if err != nil { + return nil, err + } + _, err = part.Write(blobBytes) + if err != nil { + return nil, err + } + err = writer.Close() + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", "https://api.thehive.ai/api/v2/task/sync", body) + if err != nil { + return nil, err + } + + start := time.Now() + defer func() { + duration := time.Since(start) + hiveAPIDuration.Observe(duration.Seconds()) + }() + + req.Header.Set("Authorization", fmt.Sprintf("Token %s", hal.ApiToken)) + req.Header.Add("Content-Type", writer.FormDataContentType()) + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "indigo-automod/"+versioninfo.Short()) + + req = req.WithContext(ctx) + res, err := hal.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("HiveAI request failed: %v", err) + } + defer res.Body.Close() + + hiveAPICount.WithLabelValues(fmt.Sprint(res.StatusCode)).Inc() + if res.StatusCode != 200 { + return nil, fmt.Errorf("HiveAI request failed statusCode=%d", res.StatusCode) + } + + respBytes, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read HiveAI resp body: %v", err) + } + + var respObj HiveAIResp + if err := json.Unmarshal(respBytes, &respObj); err != nil { + return nil, fmt.Errorf("failed to parse HiveAI resp JSON: %v", err) + } + slog.Info("hive-ai-response", "cid", blob.Ref.String(), "obj", respObj) + return respObj.SummarizeLabels(), nil +} diff --git a/automod/visual/hiveai_rule.go b/automod/visual/hiveai_rule.go new file mode 100644 index 000000000..850ee83b1 --- /dev/null +++ b/automod/visual/hiveai_rule.go @@ -0,0 +1,55 @@ +package visual + +import ( + "strings" + "time" + + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/helpers" + lexutil "github.com/bluesky-social/indigo/lex/util" +) + +func (hal *HiveAIClient) HiveLabelBlobRule(c *automod.RecordContext, blob lexutil.LexBlob, data []byte) error { + + if !strings.HasPrefix(blob.MimeType, "image/") { + return nil + } + + var prescreenResult string + if hal.PreScreenClient != nil { + val, err := hal.PreScreenClient.PreScreenImage(c.Ctx, data) + if err != nil { + c.Logger.Info("prescreen-request-error", "err", err) + } else { + prescreenResult = val + c.Logger.Info("prescreen-request", "uri", c.RecordOp.ATURI(), "result", prescreenResult) + } + } + + labels, err := hal.LabelBlob(c.Ctx, blob, data) + if err != nil { + return err + } + + if hal.PreScreenClient != nil { + if prescreenResult == "sfw" { + if len(labels) > 0 { + c.Logger.Info("prescreen-safe-failure", "uri", c.RecordOp.ATURI(), "labels", labels, "result", prescreenResult) + } else { + c.Logger.Info("prescreen-safe-success", "uri", c.RecordOp.ATURI()) + } + } + } + + for _, l := range labels { + // NOTE: experimenting with profile reporting for new accounts + if l == "sexual" && c.RecordOp.Collection.String() == "app.bsky.actor.profile" && helpers.AccountIsYoungerThan(&c.AccountContext, 2*24*time.Hour) { + c.ReportRecord(automod.ReportReasonSexual, "possible sexual profile (not labeled yet)") + c.Logger.Info("skipping record label", "label", l, "reason", "sexual-profile-experiment") + } else { + c.AddRecordLabel(l) + } + } + + return nil +} diff --git a/automod/visual/hiveai_test.go b/automod/visual/hiveai_test.go new file mode 100644 index 000000000..3e79aa269 --- /dev/null +++ b/automod/visual/hiveai_test.go @@ -0,0 +1,42 @@ +package visual + +import ( + "encoding/json" + "io" + "os" + "reflect" + "testing" +) + +func TestHiveParse(t *testing.T) { + file, err := os.Open("testdata/hiveai_resp_example.json") + if err != nil { + t.Fatal(err) + } + + respBytes, err := io.ReadAll(file) + if err != nil { + t.Fatal(err) + } + + var respObj HiveAIResp + if err := json.Unmarshal(respBytes, &respObj); err != nil { + t.Fatal(err) + } + + classes := respObj.Status[0].Response.Output[0].Classes + if len(classes) <= 10 { + t.Fatal("didn't get expected class count") + } + for _, c := range classes { + if c.Class == "" || c.Score == 0.0 { + t.Fatal("got null/empty class in resp") + } + } + + labels := respObj.SummarizeLabels() + expected := []string{"porn"} + if !reflect.DeepEqual(labels, expected) { + t.Fatal("didn't summarize to expected labels") + } +} diff --git a/automod/visual/metrics.go b/automod/visual/metrics.go new file mode 100644 index 000000000..bca5571ec --- /dev/null +++ b/automod/visual/metrics.go @@ -0,0 +1,26 @@ +package visual + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var hiveAPIDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "automod_hive_api_duration_sec", + Help: "Duration of Hive image auto-labeling API calls", +}) + +var hiveAPICount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_hive_api_count", + Help: "Number of Hive image auto-labeling API calls, by HTTP status code", +}, []string{"status"}) + +var abyssAPIDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "automod_abyss_api_duration_sec", + Help: "Duration of abyss image scanning API call", +}) + +var abyssAPICount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "automod_abyss_api_count", + Help: "Number of abyss image scanning API calls, by HTTP status code", +}, []string{"status"}) diff --git a/automod/visual/prescreen.go b/automod/visual/prescreen.go new file mode 100644 index 000000000..1bb6cfe25 --- /dev/null +++ b/automod/visual/prescreen.go @@ -0,0 +1,130 @@ +package visual + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "sync" + "time" +) + +const failureThresh = 10 + +type PreScreenClient struct { + Host string + Token string + + breakerEOL time.Time + breakerLk sync.Mutex + failures int + + c *http.Client +} + +func NewPreScreenClient(host, token string) *PreScreenClient { + c := &http.Client{ + Timeout: time.Second * 5, + } + + return &PreScreenClient{ + Host: host, + Token: token, + c: c, + } +} + +func (c *PreScreenClient) available() bool { + c.breakerLk.Lock() + defer c.breakerLk.Unlock() + if c.breakerEOL.IsZero() { + return true + } + + if time.Now().After(c.breakerEOL) { + c.breakerEOL = time.Time{} + return true + } + + return false +} + +func (c *PreScreenClient) recordCallResult(success bool) { + c.breakerLk.Lock() + defer c.breakerLk.Unlock() + if !c.breakerEOL.IsZero() { + return + } + + if success { + c.failures = 0 + } else { + c.failures++ + if c.failures > failureThresh { + c.breakerEOL = time.Now().Add(time.Minute) + c.failures = 0 + } + } +} + +func (c *PreScreenClient) PreScreenImage(ctx context.Context, blob []byte) (string, error) { + if !c.available() { + return "", fmt.Errorf("pre-screening temporarily unavailable") + } + + res, err := c.checkImage(ctx, blob) + if err != nil { + c.recordCallResult(false) + return "", err + } + + c.recordCallResult(true) + return res, nil +} + +type PreScreenResult struct { + Result string `json:"result"` +} + +func (c *PreScreenClient) checkImage(ctx context.Context, data []byte) (string, error) { + url := c.Host + "/predict" + + body := new(bytes.Buffer) + writer := multipart.NewWriter(body) + + part, err := writer.CreateFormFile("files", "image") + if err != nil { + return "", err + } + + part.Write(data) + + if err := writer.Close(); err != nil { + return "", err + } + + req, err := http.NewRequest("POST", url, body) + if err != nil { + return "", err + } + + req = req.WithContext(ctx) + + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Authorization", "Bearer "+c.Token) + + resp, err := c.c.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var out PreScreenResult + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", err + } + + return out.Result, nil +} diff --git a/automod/visual/testdata/hiveai_resp_example.json b/automod/visual/testdata/hiveai_resp_example.json new file mode 100644 index 000000000..2315fa9d0 --- /dev/null +++ b/automod/visual/testdata/hiveai_resp_example.json @@ -0,0 +1,401 @@ +{ + "id": "02122580-c37f-11ed-81d2-000000000000", + "code": 200, + "project_id": 12345, + "user_id": 12345, + "created_on": "2023-03-15T22:16:18.408Z", + "status": [ + { + "status": { + "code": "0", + "message": "SUCCESS" + }, + "response": { + "input": { + "id": "02122580-c37f-11ed-81d2-000000000000", + "charge": 0.003, + "model": "mod55_dense", + "model_version": 1, + "model_type": "CATEGORIZATION", + "created_on": "2023-03-15T22:16:18.136Z", + "media": { + "url": null, + "filename": "bafkreiam7k6mvkyuoybq4ynhljvj5xa75sdbhjbolzjf5j2udx7vj5gnsy", + "type": "PHOTO", + "mime_type": "jpeg", + "mimetype": "image/jpeg", + "width": 800, + "height": 800, + "num_frames": 1, + "duration": 0 + }, + "user_id": 12345, + "project_id": 12345, + "config_version": 1, + "config_tag": "default" + }, + "output": [ + { + "time": 0, + "classes": [ + { + "class": "general_not_nsfw_not_suggestive", + "score": 0.9998097218132356 + }, + { + "class": "general_nsfw", + "score": 8.857344804177162e-5 + }, + { + "class": "general_suggestive", + "score": 0.00010170473872266839 + }, + { + "class": "no_female_underwear", + "score": 0.9999923079040384 + }, + { + "class": "yes_female_underwear", + "score": 7.692095961599136e-6 + }, + { + "class": "no_male_underwear", + "score": 0.9999984904867634 + }, + { + "class": "yes_male_underwear", + "score": 1.5095132367094679e-6 + }, + { + "class": "no_sex_toy", + "score": 0.9999970970762551 + }, + { + "class": "yes_sex_toy", + "score": 2.9029237450490604e-6 + }, + { + "class": "no_female_nudity", + "score": 0.9999739028909301 + }, + { + "class": "yes_female_nudity", + "score": 2.60971090699536e-5 + }, + { + "class": "no_male_nudity", + "score": 0.9999711373083747 + }, + { + "class": "yes_male_nudity", + "score": 2.8862691625255323e-5 + }, + { + "class": "no_female_swimwear", + "score": 0.9999917609899659 + }, + { + "class": "yes_female_swimwear", + "score": 8.239010034025379e-6 + }, + { + "class": "no_male_shirtless", + "score": 0.9999583350744331 + }, + { + "class": "yes_male_shirtless", + "score": 4.166492556688088e-5 + }, + { + "class": "no_text", + "score": 0.9958378716447616 + }, + { + "class": "text", + "score": 0.0041621283552384265 + }, + { + "class": "animated", + "score": 0.46755478950048235 + }, + { + "class": "hybrid", + "score": 0.0011440363434524984 + }, + { + "class": "natural", + "score": 0.5313011741560651 + }, + { + "class": "animated_gun", + "score": 2.0713000782979496e-5 + }, + { + "class": "gun_in_hand", + "score": 1.5844730446534659e-6 + }, + { + "class": "gun_not_in_hand", + "score": 1.0338973818006654e-6 + }, + { + "class": "no_gun", + "score": 0.9999766686287906 + }, + { + "class": "culinary_knife_in_hand", + "score": 3.8063500083369785e-6 + }, + { + "class": "culinary_knife_not_in_hand", + "score": 7.94057948996249e-7 + }, + { + "class": "knife_in_hand", + "score": 4.5578955723278505e-7 + }, + { + "class": "knife_not_in_hand", + "score": 3.842124714748908e-7 + }, + { + "class": "no_knife", + "score": 0.999994559590014 + }, + { + "class": "a_little_bloody", + "score": 2.1317745626539786e-7 + }, + { + "class": "no_blood", + "score": 0.9999793341236429 + }, + { + "class": "other_blood", + "score": 2.0322054269591763e-5 + }, + { + "class": "very_bloody", + "score": 1.306446309561673e-7 + }, + { + "class": "no_pills", + "score": 0.9999989592376954 + }, + { + "class": "yes_pills", + "score": 1.0407623044588633e-6 + }, + { + "class": "no_smoking", + "score": 0.9999939101969173 + }, + { + "class": "yes_smoking", + "score": 6.089803082758281e-6 + }, + { + "class": "illicit_injectables", + "score": 6.925695592003094e-7 + }, + { + "class": "medical_injectables", + "score": 8.587808234452378e-7 + }, + { + "class": "no_injectables", + "score": 0.9999984486496174 + }, + { + "class": "no_nazi", + "score": 0.9999987449628097 + }, + { + "class": "yes_nazi", + "score": 1.2550371902234279e-6 + }, + { + "class": "no_kkk", + "score": 0.999999762417549 + }, + { + "class": "yes_kkk", + "score": 2.3758245111050425e-7 + }, + { + "class": "no_middle_finger", + "score": 0.9999881515231847 + }, + { + "class": "yes_middle_finger", + "score": 1.184847681536747e-5 + }, + { + "class": "no_terrorist", + "score": 0.9999998870793229 + }, + { + "class": "yes_terrorist", + "score": 1.1292067715380635e-7 + }, + { + "class": "no_overlay_text", + "score": 0.9996453363440359 + }, + { + "class": "yes_overlay_text", + "score": 0.0003546636559640924 + }, + { + "class": "no_sexual_activity", + "score": 0.9999563580374798 + }, + { + "class": "yes_sexual_activity", + "score": 0.99, + "realScore": 4.364196252012032e-5 + }, + { + "class": "hanging", + "score": 3.6435135762510905e-7 + }, + { + "class": "no_hanging_no_noose", + "score": 0.9999980779196416 + }, + { + "class": "noose", + "score": 1.5577290007796094e-6 + }, + { + "class": "no_realistic_nsfw", + "score": 0.9999944341007805 + }, + { + "class": "yes_realistic_nsfw", + "score": 5.565899219571182e-6 + }, + { + "class": "animated_corpse", + "score": 5.276802046755426e-7 + }, + { + "class": "human_corpse", + "score": 2.5449360984211012e-8 + }, + { + "class": "no_corpse", + "score": 0.9999994468704343 + }, + { + "class": "no_self_harm", + "score": 0.9999994515625507 + }, + { + "class": "yes_self_harm", + "score": 5.484374493605692e-7 + }, + { + "class": "no_drawing", + "score": 0.9978276028816608 + }, + { + "class": "yes_drawing", + "score": 0.0021723971183392485 + }, + { + "class": "no_emaciated_body", + "score": 0.9999998146500432 + }, + { + "class": "yes_emaciated_body", + "score": 1.853499568724518e-7 + }, + { + "class": "no_child_present", + "score": 0.9999970498515446 + }, + { + "class": "yes_child_present", + "score": 2.950148455380443e-6 + }, + { + "class": "no_sexual_intent", + "score": 0.9999963861546292 + }, + { + "class": "yes_sexual_intent", + "score": 3.613845370766111e-6 + }, + { + "class": "animal_genitalia_and_human", + "score": 2.255472023465222e-8 + }, + { + "class": "animal_genitalia_only", + "score": 4.6783185199931176e-7 + }, + { + "class": "animated_animal_genitalia", + "score": 6.707857419436447e-7 + }, + { + "class": "no_animal_genitalia", + "score": 0.9999988388276858 + }, + { + "class": "no_gambling", + "score": 0.9999960939687145 + }, + { + "class": "yes_gambling", + "score": 3.906031285604864e-6 + }, + { + "class": "no_undressed", + "score": 0.99999923356218 + }, + { + "class": "yes_undressed", + "score": 7.664378199789045e-7 + }, + { + "class": "no_confederate", + "score": 0.9999925456900376 + }, + { + "class": "yes_confederate", + "score": 7.454309962453175e-6 + }, + { + "class": "animated_alcohol", + "score": 1.8109949948066074e-6 + }, + { + "class": "no_alcohol", + "score": 0.9999916620957963 + }, + { + "class": "yes_alcohol", + "score": 5.88781463445443e-6 + }, + { + "class": "yes_drinking_alcohol", + "score": 6.390945746578106e-7 + }, + { + "class": "no_religious_icon", + "score": 0.9999862158580689 + }, + { + "class": "yes_religious_icon", + "score": 1.3784141931119298e-5 + } + ] + } + ] + } + } + ], + "from_cache": false +} diff --git a/backfill/backfill.go b/backfill/backfill.go new file mode 100644 index 000000000..dc59ae453 --- /dev/null +++ b/backfill/backfill.go @@ -0,0 +1,633 @@ +package backfill + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "strings" + "sync" + "time" + + "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/repo" + "github.com/bluesky-social/indigo/repomgr" + + "github.com/ipfs/go-cid" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "golang.org/x/sync/semaphore" + "golang.org/x/time/rate" +) + +// Job is an interface for a backfill job +type Job interface { + Repo() string + State() string + Rev() string + SetState(ctx context.Context, state string) error + SetRev(ctx context.Context, rev string) error + RetryCount() int + + // BufferOps buffers the given operations and returns true if the operations + // were buffered. + // The given operations move the repo from since to rev. + BufferOps(ctx context.Context, since *string, rev string, ops []*BufferedOp) (bool, error) + // FlushBufferedOps calls the given callback for each buffered operation + // Once done it clears the buffer and marks the job as "complete" + // Allowing the Job interface to abstract away the details of how buffered + // operations are stored and/or locked + FlushBufferedOps(ctx context.Context, cb func(kind repomgr.EventKind, rev, path string, rec *[]byte, cid *cid.Cid) error) error + + ClearBufferedOps(ctx context.Context) error +} + +// Store is an interface for a backfill store which holds Jobs +type Store interface { + // BufferOp buffers an operation for a job and returns true if the operation was buffered + // If the operation was not buffered, it returns false and an error (ErrJobNotFound or ErrJobComplete) + GetJob(ctx context.Context, repo string) (Job, error) + GetNextEnqueuedJob(ctx context.Context) (Job, error) + UpdateRev(ctx context.Context, repo, rev string) error + + EnqueueJob(ctx context.Context, repo string) error + EnqueueJobWithState(ctx context.Context, repo string, state string) error + + PurgeRepo(ctx context.Context, repo string) error +} + +// Backfiller is a struct which handles backfilling a repo +type Backfiller struct { + Name string + HandleCreateRecord func(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error + HandleUpdateRecord func(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error + HandleDeleteRecord func(ctx context.Context, repo string, rev string, path string) error + Store Store + + // Number of backfills to process in parallel + ParallelBackfills int + // Number of records to process in parallel for each backfill + ParallelRecordCreates int + // Prefix match for records to backfill i.e. app.bsky.feed.app/ + // If empty, all records will be backfilled + NSIDFilter string + RelayHost string + + syncLimiter *rate.Limiter + + magicHeaderKey string + magicHeaderVal string + + tryRelayRepoFetch bool + + stop chan chan struct{} + + Directory identity.Directory +} + +var ( + // StateEnqueued is the state of a backfill job when it is first created + StateEnqueued = "enqueued" + // StateInProgress is the state of a backfill job when it is being processed + StateInProgress = "in_progress" + // StateComplete is the state of a backfill job when it has been processed + StateComplete = "complete" +) + +// ErrJobComplete is returned when trying to buffer an op for a job that is complete +var ErrJobComplete = errors.New("job is complete") + +// ErrJobNotFound is returned when trying to buffer an op for a job that doesn't exist +var ErrJobNotFound = errors.New("job not found") + +// ErrEventGap is returned when an event is received with a since that doesn't match the current rev +var ErrEventGap = fmt.Errorf("buffered event revs did not line up") + +// ErrAlreadyProcessed is returned when attempting to buffer an event that has already been accounted for (rev older than current) +var ErrAlreadyProcessed = fmt.Errorf("event already accounted for") + +var tracer = otel.Tracer("backfiller") + +type BackfillOptions struct { + ParallelBackfills int + ParallelRecordCreates int + NSIDFilter string + SyncRequestsPerSecond int + RelayHost string +} + +func DefaultBackfillOptions() *BackfillOptions { + return &BackfillOptions{ + ParallelBackfills: 10, + ParallelRecordCreates: 100, + NSIDFilter: "", + SyncRequestsPerSecond: 2, + RelayHost: "https://bsky.network", + } +} + +// NewBackfiller creates a new Backfiller +func NewBackfiller( + name string, + store Store, + handleCreate func(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error, + handleUpdate func(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error, + handleDelete func(ctx context.Context, repo string, rev string, path string) error, + opts *BackfillOptions, +) *Backfiller { + if opts == nil { + opts = DefaultBackfillOptions() + } + + // Convert wss:// or ws:// to https:// or http:// + if strings.HasPrefix(opts.RelayHost, "wss://") { + opts.RelayHost = "https://" + opts.RelayHost[6:] + } else if strings.HasPrefix(opts.RelayHost, "ws://") { + opts.RelayHost = "http://" + opts.RelayHost[5:] + } + + return &Backfiller{ + Name: name, + Store: store, + HandleCreateRecord: handleCreate, + HandleUpdateRecord: handleUpdate, + HandleDeleteRecord: handleDelete, + ParallelBackfills: opts.ParallelBackfills, + ParallelRecordCreates: opts.ParallelRecordCreates, + NSIDFilter: opts.NSIDFilter, + syncLimiter: rate.NewLimiter(rate.Limit(opts.SyncRequestsPerSecond), 1), + RelayHost: opts.RelayHost, + stop: make(chan chan struct{}, 1), + Directory: identity.DefaultDirectory(), + } +} + +// Start starts the backfill processor routine +func (b *Backfiller) Start() { + ctx := context.Background() + + log := slog.With("source", "backfiller", "name", b.Name) + log.Info("starting backfill processor") + + sem := semaphore.NewWeighted(int64(b.ParallelBackfills)) + + for { + select { + case stopped := <-b.stop: + log.Info("stopping backfill processor") + sem.Acquire(ctx, int64(b.ParallelBackfills)) + close(stopped) + return + default: + } + + // Get the next job + job, err := b.Store.GetNextEnqueuedJob(ctx) + if err != nil { + log.Error("failed to get next enqueued job", "error", err) + time.Sleep(1 * time.Second) + continue + } else if job == nil { + time.Sleep(1 * time.Second) + continue + } + + log := log.With("repo", job.Repo()) + + // Mark the backfill as "in progress" + err = job.SetState(ctx, StateInProgress) + if err != nil { + log.Error("failed to set job state", "error", err) + continue + } + + sem.Acquire(ctx, 1) + go func(j Job) { + defer sem.Release(1) + newState, err := b.BackfillRepo(ctx, j) + if err != nil { + log.Error("failed to backfill repo", "error", err) + } + if newState != "" { + if sserr := j.SetState(ctx, newState); sserr != nil { + log.Error("failed to set job state", "error", sserr) + } + + if strings.HasPrefix(newState, "failed") { + // Clear buffered ops + if err := j.ClearBufferedOps(ctx); err != nil { + log.Error("failed to clear buffered ops", "error", err) + } + } + } + backfillJobsProcessed.WithLabelValues(b.Name).Inc() + }(job) + } +} + +// Stop stops the backfill processor +func (b *Backfiller) Stop(ctx context.Context) error { + log := slog.With("source", "backfiller", "name", b.Name) + log.Info("stopping backfill processor") + stopped := make(chan struct{}) + b.stop <- stopped + select { + case <-stopped: + log.Info("backfill processor stopped") + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// FlushBuffer processes buffered operations for a job +func (b *Backfiller) FlushBuffer(ctx context.Context, job Job) int { + ctx, span := tracer.Start(ctx, "FlushBuffer") + defer span.End() + log := slog.With("source", "backfiller_buffer_flush", "repo", job.Repo()) + + processed := 0 + + repo := job.Repo() + + // Flush buffered operations, clear the buffer, and mark the job as "complete" + // Clearing and marking are handled by the job interface + err := job.FlushBufferedOps(ctx, func(kind repomgr.EventKind, rev, path string, rec *[]byte, cid *cid.Cid) error { + switch kind { + case repomgr.EvtKindCreateRecord: + err := b.HandleCreateRecord(ctx, repo, rev, path, rec, cid) + if err != nil { + log.Error("failed to handle create record", "error", err) + } + case repomgr.EvtKindUpdateRecord: + err := b.HandleUpdateRecord(ctx, repo, rev, path, rec, cid) + if err != nil { + log.Error("failed to handle update record", "error", err) + } + case repomgr.EvtKindDeleteRecord: + err := b.HandleDeleteRecord(ctx, repo, rev, path) + if err != nil { + log.Error("failed to handle delete record", "error", err) + } + } + backfillOpsBuffered.WithLabelValues(b.Name).Dec() + processed++ + return nil + }) + if err != nil { + log.Error("failed to flush buffered ops", "error", err) + if errors.Is(err, ErrEventGap) { + if sserr := job.SetState(ctx, StateEnqueued); sserr != nil { + log.Error("failed to reset job state after failed buffer flush", "error", sserr) + } + // TODO: need to re-queue this job for later + return processed + } + } + + // Mark the job as "complete" + err = job.SetState(ctx, StateComplete) + if err != nil { + log.Error("failed to set job state", "error", err) + } + + return processed +} + +type recordQueueItem struct { + recordPath string + nodeCid cid.Cid +} + +type recordResult struct { + recordPath string + err error +} + +type FetchRepoError struct { + StatusCode int + Status string +} + +func (e *FetchRepoError) Error() string { + reason := "unknown error" + if e.StatusCode == http.StatusBadRequest { + reason = "repo not found" + } else { + reason = e.Status + } + return fmt.Sprintf("failed to get repo: %s (%d)", reason, e.StatusCode) +} + +// Fetches a repo CAR file over HTTP from the indicated host. If successful, parses the CAR and returns repo.Repo +func (b *Backfiller) fetchRepo(ctx context.Context, did, since, host string) (*repo.Repo, error) { + url := fmt.Sprintf("%s/xrpc/com.atproto.sync.getRepo?did=%s", host, did) + + if since != "" { + url = url + fmt.Sprintf("&since=%s", since) + } + + // GET and CAR decode the body + client := &http.Client{ + Transport: otelhttp.NewTransport(http.DefaultTransport), + Timeout: 600 * time.Second, + } + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Accept", "application/vnd.ipld.car") + req.Header.Set("User-Agent", fmt.Sprintf("atproto-backfill-%s/0.0.1", b.Name)) + if b.magicHeaderKey != "" && b.magicHeaderVal != "" { + req.Header.Set(b.magicHeaderKey, b.magicHeaderVal) + } + + b.syncLimiter.Wait(ctx) + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, &FetchRepoError{ + StatusCode: resp.StatusCode, + Status: resp.Status, + } + } + + instrumentedReader := instrumentedReader{ + source: resp.Body, + counter: backfillBytesProcessed.WithLabelValues(b.Name), + } + + defer instrumentedReader.Close() + + repo, err := repo.ReadRepoFromCar(ctx, instrumentedReader) + if err != nil { + return nil, fmt.Errorf("failed to parse repo from CAR file: %w", err) + } + return repo, nil +} + +// BackfillRepo backfills a repo +func (b *Backfiller) BackfillRepo(ctx context.Context, job Job) (string, error) { + ctx, span := tracer.Start(ctx, "BackfillRepo") + defer span.End() + + start := time.Now() + + repoDID := job.Repo() + + log := slog.With("source", "backfiller_backfill_repo", "repo", repoDID) + if job.RetryCount() > 0 { + log = log.With("retry_count", job.RetryCount()) + } + log.Info(fmt.Sprintf("processing backfill for %s", repoDID)) + + var r *repo.Repo + if b.tryRelayRepoFetch { + rr, err := b.fetchRepo(ctx, repoDID, job.Rev(), b.RelayHost) + if err != nil { + slog.Warn("repo CAR fetch from relay failed", "did", repoDID, "since", job.Rev(), "relayHost", b.RelayHost, "err", err) + } else { + r = rr + } + } + + if r == nil { + ident, err := b.Directory.LookupDID(ctx, syntax.DID(repoDID)) + if err != nil { + return "failed resolving DID to PDS repo", fmt.Errorf("resolving DID for PDS repo fetch: %w", err) + } + pdsHost := ident.PDSEndpoint() + if pdsHost == "" { + return "DID document missing PDS endpoint", fmt.Errorf("no PDS endpoint for DID: %s", repoDID) + } + + r, err = b.fetchRepo(ctx, repoDID, job.Rev(), pdsHost) + if err != nil { + slog.Warn("repo CAR fetch from PDS failed", "did", repoDID, "since", job.Rev(), "pdsHost", pdsHost, "err", err) + rfe, ok := err.(*FetchRepoError) + if ok { + return fmt.Sprintf("failed to fetch repo CAR from PDS (http %d:%s)", rfe.StatusCode, rfe.Status), err + } + return "failed to fetch repo CAR from PDS", err + } + } + + numRecords := 0 + numRoutines := b.ParallelRecordCreates + recordQueue := make(chan recordQueueItem, numRoutines) + recordResults := make(chan recordResult, numRoutines) + + // Producer routine + go func() { + defer close(recordQueue) + if err := r.ForEach(ctx, b.NSIDFilter, func(recordPath string, nodeCid cid.Cid) error { + numRecords++ + recordQueue <- recordQueueItem{recordPath: recordPath, nodeCid: nodeCid} + return nil + }); err != nil { + log.Error("failed to iterate records in repo", "err", err) + } + }() + + rev := r.SignedCommit().Rev + + // Consumer routines + wg := sync.WaitGroup{} + for i := 0; i < numRoutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for item := range recordQueue { + blk, err := r.Blockstore().Get(ctx, item.nodeCid) + if err != nil { + recordResults <- recordResult{recordPath: item.recordPath, err: fmt.Errorf("failed to get blocks for record: %w", err)} + continue + } + + raw := blk.RawData() + + err = b.HandleCreateRecord(ctx, repoDID, rev, item.recordPath, &raw, &item.nodeCid) + if err != nil { + recordResults <- recordResult{recordPath: item.recordPath, err: fmt.Errorf("failed to handle create record: %w", err)} + continue + } + + backfillRecordsProcessed.WithLabelValues(b.Name).Inc() + recordResults <- recordResult{recordPath: item.recordPath, err: err} + } + }() + } + + resultWG := sync.WaitGroup{} + resultWG.Add(1) + // Handle results + go func() { + defer resultWG.Done() + for result := range recordResults { + if result.err != nil { + log.Error("Error processing record", "record", result.recordPath, "error", result.err) + } + } + }() + + wg.Wait() + close(recordResults) + resultWG.Wait() + + if err := job.SetRev(ctx, r.SignedCommit().Rev); err != nil { + log.Error("failed to update rev after backfilling repo", "err", err) + } + + // Process buffered operations, marking the job as "complete" when done + numProcessed := b.FlushBuffer(ctx, job) + + log.Info("backfill complete", + "buffered_records_processed", numProcessed, + "records_backfilled", numRecords, + "duration", time.Since(start), + ) + + return StateComplete, nil +} + +const trust = true + +func (bf *Backfiller) getRecord(ctx context.Context, r *repo.Repo, op *atproto.SyncSubscribeRepos_RepoOp) (cid.Cid, *[]byte, error) { + if trust { + if op.Cid == nil { + return cid.Undef, nil, fmt.Errorf("op had no cid set") + } + + c := (cid.Cid)(*op.Cid) + blk, err := r.Blockstore().Get(ctx, c) + if err != nil { + return cid.Undef, nil, err + } + + raw := blk.RawData() + + return c, &raw, nil + } else { + return r.GetRecordBytes(ctx, op.Path) + } +} + +func (bf *Backfiller) HandleEvent(ctx context.Context, evt *atproto.SyncSubscribeRepos_Commit) error { + r, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks)) + if err != nil { + return fmt.Errorf("failed to read event repo: %w", err) + } + + ops := make([]*BufferedOp, 0, len(evt.Ops)) + for _, op := range evt.Ops { + kind := repomgr.EventKind(op.Action) + switch kind { + case repomgr.EvtKindCreateRecord, repomgr.EvtKindUpdateRecord: + cc, rec, err := bf.getRecord(ctx, r, op) + if err != nil { + return fmt.Errorf("getting record failed (%s,%s): %w", op.Action, op.Path, err) + } + ops = append(ops, &BufferedOp{ + Kind: kind, + Path: op.Path, + Record: rec, + Cid: &cc, + }) + case repomgr.EvtKindDeleteRecord: + ops = append(ops, &BufferedOp{ + Kind: kind, + Path: op.Path, + }) + default: + return fmt.Errorf("invalid op action: %q", op.Action) + } + } + + if evt.Since == nil { + // The first event for a repo will have a nil since, we can enqueue the repo as "complete" to avoid fetching the empty repo + if err := bf.Store.EnqueueJobWithState(ctx, evt.Repo, StateComplete); err != nil { + return fmt.Errorf("failed to enqueue job with state for repo %q: %w", evt.Repo, err) + } + } + + buffered, err := bf.BufferOps(ctx, evt.Repo, evt.Since, evt.Rev, ops) + if err != nil { + if errors.Is(err, ErrAlreadyProcessed) { + return nil + } + return fmt.Errorf("buffer ops failed: %w", err) + } + + if buffered { + return nil + } + + for _, op := range ops { + switch op.Kind { + case repomgr.EvtKindCreateRecord: + if err := bf.HandleCreateRecord(ctx, evt.Repo, evt.Rev, op.Path, op.Record, op.Cid); err != nil { + return fmt.Errorf("create record failed: %w", err) + } + case repomgr.EvtKindUpdateRecord: + if err := bf.HandleUpdateRecord(ctx, evt.Repo, evt.Rev, op.Path, op.Record, op.Cid); err != nil { + return fmt.Errorf("update record failed: %w", err) + } + case repomgr.EvtKindDeleteRecord: + if err := bf.HandleDeleteRecord(ctx, evt.Repo, evt.Rev, op.Path); err != nil { + return fmt.Errorf("delete record failed: %w", err) + } + } + } + + if err := bf.Store.UpdateRev(ctx, evt.Repo, evt.Rev); err != nil { + return fmt.Errorf("failed to update rev: %w", err) + } + + return nil +} + +func (bf *Backfiller) BufferOp(ctx context.Context, repo string, since *string, rev string, kind repomgr.EventKind, path string, rec *[]byte, cid *cid.Cid) (bool, error) { + return bf.BufferOps(ctx, repo, since, rev, []*BufferedOp{{ + Path: path, + Kind: kind, + Record: rec, + Cid: cid, + }}) +} + +func (bf *Backfiller) BufferOps(ctx context.Context, repo string, since *string, rev string, ops []*BufferedOp) (bool, error) { + j, err := bf.Store.GetJob(ctx, repo) + if err != nil { + if !errors.Is(err, ErrJobNotFound) { + return false, err + } + if qerr := bf.Store.EnqueueJob(ctx, repo); qerr != nil { + return false, fmt.Errorf("failed to enqueue job for unknown repo: %w", qerr) + } + + nj, err := bf.Store.GetJob(ctx, repo) + if err != nil { + return false, err + } + + j = nj + } + + return j.BufferOps(ctx, since, rev, ops) +} + +// MaxRetries is the maximum number of times to retry a backfill job +var MaxRetries = 10 + +func computeExponentialBackoff(attempt int) time.Duration { + return time.Duration(1<= 5 && ts.creates >= 1 && ts.updates >= 1 { + ts.lk.Unlock() + break + } + ts.lk.Unlock() + time.Sleep(100 * time.Millisecond) + } + + bf.Stop() + + slog.Info("shutting down") + */ +} + +func (ts *testState) handleCreate(ctx context.Context, repo string, path string, rec typegen.CBORMarshaler, cid *cid.Cid) error { + slog.Info("got create", "repo", repo, "path", path) + ts.lk.Lock() + ts.creates++ + ts.lk.Unlock() + return nil +} + +func (ts *testState) handleUpdate(ctx context.Context, repo string, path string, rec typegen.CBORMarshaler, cid *cid.Cid) error { + slog.Info("got update", "repo", repo, "path", path) + ts.lk.Lock() + ts.updates++ + ts.lk.Unlock() + return nil +} + +func (ts *testState) handleDelete(ctx context.Context, repo string, path string) error { + slog.Info("got delete", "repo", repo, "path", path) + ts.lk.Lock() + ts.deletes++ + ts.lk.Unlock() + return nil +} diff --git a/backfill/gormstore.go b/backfill/gormstore.go new file mode 100644 index 000000000..a3816fdc8 --- /dev/null +++ b/backfill/gormstore.go @@ -0,0 +1,432 @@ +package backfill + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/bluesky-social/indigo/repomgr" + "github.com/ipfs/go-cid" + "gorm.io/gorm" +) + +type Gormjob struct { + repo string + state string + rev string + + lk sync.Mutex + bufferedOps []*opSet + + dbj *GormDBJob + db *gorm.DB + + createdAt time.Time + updatedAt time.Time + + retryCount int + retryAfter *time.Time +} + +type GormDBJob struct { + gorm.Model + Repo string `gorm:"unique;index"` + State string `gorm:"index:enqueued_job_idx,where:state = 'enqueued';index:retryable_job_idx,where:state like 'failed%'"` + Rev string + RetryCount int + RetryAfter *time.Time `gorm:"index:retryable_job_idx,sort:desc"` +} + +// Gormstore is a gorm-backed implementation of the Backfill Store interface +type Gormstore struct { + lk sync.RWMutex + jobs map[string]*Gormjob + + qlk sync.Mutex + taskQueue []string + + db *gorm.DB +} + +func NewGormstore(db *gorm.DB) *Gormstore { + return &Gormstore{ + jobs: make(map[string]*Gormjob), + db: db, + } +} + +func (s *Gormstore) LoadJobs(ctx context.Context) error { + s.qlk.Lock() + defer s.qlk.Unlock() + return s.loadJobs(ctx, 20_000) +} + +func (s *Gormstore) loadJobs(ctx context.Context, limit int) error { + enqueuedIndexClause := "" + retryableIndexClause := "" + + // If the DB is a SQLite DB, we can use INDEXED BY to speed up the query + if s.db.Dialector.Name() == "sqlite" { + enqueuedIndexClause = "INDEXED BY enqueued_job_idx" + retryableIndexClause = "INDEXED BY retryable_job_idx" + } + + enqueuedSelect := fmt.Sprintf(`SELECT repo FROM gorm_db_jobs %s WHERE state = 'enqueued' LIMIT ?`, enqueuedIndexClause) + retryableSelect := fmt.Sprintf(`SELECT repo FROM gorm_db_jobs %s WHERE state like 'failed%%' AND (retry_after = NULL OR retry_after < ?) LIMIT ?`, retryableIndexClause) + + var todo []string + if err := s.db.Raw(enqueuedSelect, limit).Scan(&todo).Error; err != nil { + return err + } + + if len(todo) < limit { + var moreTodo []string + if err := s.db.Raw(retryableSelect, time.Now(), limit-len(todo)).Scan(&moreTodo).Error; err != nil { + return err + } + todo = append(todo, moreTodo...) + } + + s.taskQueue = append(s.taskQueue, todo...) + + return nil +} + +func (s *Gormstore) GetOrCreateJob(ctx context.Context, repo, state string) (Job, error) { + j, err := s.getJob(ctx, repo) + if err == nil { + return j, nil + } + + if !errors.Is(err, ErrJobNotFound) { + return nil, err + } + + if err := s.createJobForRepo(repo, state); err != nil { + return nil, err + } + + return s.getJob(ctx, repo) +} + +func (s *Gormstore) EnqueueJob(ctx context.Context, repo string) error { + _, err := s.GetOrCreateJob(ctx, repo, StateEnqueued) + if err != nil { + return err + } + + s.qlk.Lock() + s.taskQueue = append(s.taskQueue, repo) + s.qlk.Unlock() + + return nil +} + +func (s *Gormstore) EnqueueJobWithState(ctx context.Context, repo, state string) error { + _, err := s.GetOrCreateJob(ctx, repo, state) + if err != nil { + return err + } + + s.qlk.Lock() + s.taskQueue = append(s.taskQueue, repo) + s.qlk.Unlock() + + return nil +} + +func (s *Gormstore) createJobForRepo(repo, state string) error { + dbj := &GormDBJob{ + Repo: repo, + State: state, + } + if err := s.db.Create(dbj).Error; err != nil { + if err == gorm.ErrDuplicatedKey { + return nil + } + return err + } + + s.lk.Lock() + defer s.lk.Unlock() + + // Convert it to an in-memory job + if _, ok := s.jobs[repo]; ok { + // The DB create should have errored if the job already existed, but just in case + return fmt.Errorf("job already exists for repo %s", repo) + } + + j := &Gormjob{ + repo: repo, + createdAt: time.Now(), + updatedAt: time.Now(), + state: state, + + dbj: dbj, + db: s.db, + } + s.jobs[repo] = j + + return nil +} + +func (j *Gormjob) BufferOps(ctx context.Context, since *string, rev string, ops []*BufferedOp) (bool, error) { + j.lk.Lock() + defer j.lk.Unlock() + + switch j.state { + case StateComplete: + return false, nil + case StateInProgress, StateEnqueued: + // keep going and buffer the op + default: + if strings.HasPrefix(j.state, "failed") { + if j.retryCount >= MaxRetries { + // Process immediately since we're out of retries + return false, nil + } + // Don't buffer the op since it'll get caught in the next retry (hopefully) + return true, nil + } + return false, fmt.Errorf("invalid job state: %q", j.state) + } + + if j.rev >= rev || (since == nil && j.rev != "") { + // we've already accounted for this event + return false, ErrAlreadyProcessed + } + + j.bufferOps(&opSet{since: since, rev: rev, ops: ops}) + return true, nil +} + +func (j *Gormjob) bufferOps(ops *opSet) { + j.bufferedOps = append(j.bufferedOps, ops) + j.updatedAt = time.Now() +} + +func (s *Gormstore) GetJob(ctx context.Context, repo string) (Job, error) { + return s.getJob(ctx, repo) +} + +func (s *Gormstore) getJob(ctx context.Context, repo string) (*Gormjob, error) { + cj := s.checkJobCache(ctx, repo) + if cj != nil { + return cj, nil + } + + return s.loadJob(ctx, repo) +} + +func (s *Gormstore) loadJob(ctx context.Context, repo string) (*Gormjob, error) { + var dbj GormDBJob + if err := s.db.Find(&dbj, "repo = ?", repo).Error; err != nil { + return nil, err + } + + if dbj.ID == 0 { + return nil, ErrJobNotFound + } + + j := &Gormjob{ + repo: dbj.Repo, + state: dbj.State, + createdAt: dbj.CreatedAt, + updatedAt: dbj.UpdatedAt, + + dbj: &dbj, + db: s.db, + + retryCount: dbj.RetryCount, + retryAfter: dbj.RetryAfter, + } + s.lk.Lock() + defer s.lk.Unlock() + // would imply a race condition + exist, ok := s.jobs[repo] + if ok { + return exist, nil + } + s.jobs[repo] = j + return j, nil +} + +func (s *Gormstore) checkJobCache(ctx context.Context, repo string) *Gormjob { + s.lk.RLock() + defer s.lk.RUnlock() + + j, ok := s.jobs[repo] + if !ok || j == nil { + return nil + } + return j +} + +func (s *Gormstore) GetNextEnqueuedJob(ctx context.Context) (Job, error) { + s.qlk.Lock() + defer s.qlk.Unlock() + if len(s.taskQueue) == 0 { + if err := s.loadJobs(ctx, 1000); err != nil { + return nil, err + } + + if len(s.taskQueue) == 0 { + return nil, nil + } + } + + for len(s.taskQueue) > 0 { + first := s.taskQueue[0] + s.taskQueue = s.taskQueue[1:] + + j, err := s.getJob(ctx, first) + if err != nil { + return nil, err + } + + shouldRetry := strings.HasPrefix(j.State(), "failed") && j.retryAfter != nil && time.Now().After(*j.retryAfter) + + if j.State() == StateEnqueued || shouldRetry { + return j, nil + } + } + return nil, nil +} + +func (j *Gormjob) Repo() string { + return j.repo +} + +func (j *Gormjob) State() string { + j.lk.Lock() + defer j.lk.Unlock() + + return j.state +} + +func (j *Gormjob) SetRev(ctx context.Context, r string) error { + j.lk.Lock() + defer j.lk.Unlock() + + j.rev = r + j.updatedAt = time.Now() + + j.dbj.Rev = r + j.dbj.UpdatedAt = j.updatedAt + + // Persist the job to the database + + return j.db.Save(j.dbj).Error +} + +func (j *Gormjob) Rev() string { + j.lk.Lock() + defer j.lk.Unlock() + + return j.rev +} + +func (j *Gormjob) SetState(ctx context.Context, state string) error { + j.lk.Lock() + defer j.lk.Unlock() + + j.state = state + j.updatedAt = time.Now() + + if strings.HasPrefix(state, "failed") { + if j.retryCount < MaxRetries { + next := time.Now().Add(computeExponentialBackoff(j.retryCount)) + j.retryAfter = &next + j.retryCount++ + } else { + j.retryAfter = nil + } + } + + // Persist the job to the database + j.dbj.State = state + return j.db.Save(j.dbj).Error +} + +func (j *Gormjob) FlushBufferedOps(ctx context.Context, fn func(kind repomgr.EventKind, rev, path string, rec *[]byte, cid *cid.Cid) error) error { + // TODO: this will block any events for this repo while this flush is ongoing, is that okay? + j.lk.Lock() + defer j.lk.Unlock() + + for _, opset := range j.bufferedOps { + if opset.rev <= j.rev { + // stale events, skip + continue + } + + if opset.since == nil { + // The first event for a repo may have a nil since + // We should process it only if the rev is empty, skip otherwise + if j.rev != "" { + continue + } + } + + if j.rev > *opset.since { + // we've already accounted for this event + continue + } + + if j.rev != *opset.since { + // we've got a discontinuity + return fmt.Errorf("event since did not match current rev (%s != %s): %w", *opset.since, j.rev, ErrEventGap) + } + + for _, op := range opset.ops { + if err := fn(op.Kind, opset.rev, op.Path, op.Record, op.Cid); err != nil { + return err + } + } + + j.rev = opset.rev + } + + j.bufferedOps = []*opSet{} + j.state = StateComplete + + return nil +} + +func (j *Gormjob) ClearBufferedOps(ctx context.Context) error { + j.lk.Lock() + defer j.lk.Unlock() + + j.bufferedOps = []*opSet{} + j.updatedAt = time.Now() + return nil +} + +func (j *Gormjob) RetryCount() int { + j.lk.Lock() + defer j.lk.Unlock() + return j.retryCount +} + +func (s *Gormstore) UpdateRev(ctx context.Context, repo, rev string) error { + j, err := s.GetJob(ctx, repo) + if err != nil { + return err + } + + return j.SetRev(ctx, rev) +} + +func (s *Gormstore) PurgeRepo(ctx context.Context, repo string) error { + if err := s.db.Exec("DELETE FROM gorm_db_jobs WHERE repo = ?", repo).Error; err != nil { + return err + } + + s.lk.Lock() + defer s.lk.Unlock() + delete(s.jobs, repo) + + return nil +} diff --git a/backfill/memstore.go b/backfill/memstore.go new file mode 100644 index 000000000..8e0aff78b --- /dev/null +++ b/backfill/memstore.go @@ -0,0 +1,242 @@ +package backfill + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/bluesky-social/indigo/repomgr" + "github.com/ipfs/go-cid" +) + +// A BufferedOp is an operation buffered while a repo is being backfilled. +type BufferedOp struct { + // Kind describes the type of operation. + Kind repomgr.EventKind + // Path contains the path the operation applies to. + Path string + // Record contains the serialized record for create and update operations. + Record *[]byte + // Cid is the CID of the record. + Cid *cid.Cid +} + +type opSet struct { + since *string + rev string + ops []*BufferedOp +} + +type Memjob struct { + repo string + state string + rev string + lk sync.Mutex + bufferedOps []*opSet + + createdAt time.Time + updatedAt time.Time +} + +// Memstore is a simple in-memory implementation of the Backfill Store interface +type Memstore struct { + lk sync.RWMutex + jobs map[string]*Memjob +} + +func NewMemstore() *Memstore { + return &Memstore{ + jobs: make(map[string]*Memjob), + } +} + +func (s *Memstore) EnqueueJob(repo string) error { + s.lk.Lock() + defer s.lk.Unlock() + + if _, ok := s.jobs[repo]; ok { + return fmt.Errorf("job already exists for repo %s", repo) + } + + j := &Memjob{ + repo: repo, + createdAt: time.Now(), + updatedAt: time.Now(), + state: StateEnqueued, + } + s.jobs[repo] = j + return nil +} + +func (s *Memstore) EnqueueJobWithState(repo, state string) error { + s.lk.Lock() + defer s.lk.Unlock() + + if _, ok := s.jobs[repo]; ok { + return fmt.Errorf("job already exists for repo %s", repo) + } + + j := &Memjob{ + repo: repo, + createdAt: time.Now(), + updatedAt: time.Now(), + state: state, + } + s.jobs[repo] = j + return nil +} + +func (s *Memstore) BufferOp(ctx context.Context, repo string, since *string, rev string, kind repomgr.EventKind, path string, rec *[]byte, cid *cid.Cid) (bool, error) { + s.lk.Lock() + + // If the job doesn't exist, we can't buffer an op for it + j, ok := s.jobs[repo] + s.lk.Unlock() + if !ok { + return false, ErrJobNotFound + } + + j.lk.Lock() + defer j.lk.Unlock() + + switch j.state { + case StateComplete: + return false, ErrJobComplete + case StateInProgress: + // keep going and buffer the op + default: + return false, nil + } + + j.bufferedOps = append(j.bufferedOps, &opSet{ + since: since, + rev: rev, + ops: []*BufferedOp{&BufferedOp{ + Path: path, + Kind: kind, + Record: rec, + Cid: cid, + }}, + }) + j.updatedAt = time.Now() + return true, nil +} + +func (j *Memjob) BufferOps(ctx context.Context, since *string, rev string, ops []*BufferedOp) (bool, error) { + j.lk.Lock() + defer j.lk.Unlock() + + switch j.state { + case StateComplete: + return false, ErrJobComplete + case StateInProgress: + // keep going and buffer the op + default: + return false, nil + } + + j.bufferedOps = append(j.bufferedOps, &opSet{ + since: since, + rev: rev, + ops: ops, + }) + j.updatedAt = time.Now() + return true, nil +} + +func (s *Memstore) GetJob(ctx context.Context, repo string) (Job, error) { + s.lk.RLock() + defer s.lk.RUnlock() + + j, ok := s.jobs[repo] + if !ok || j == nil { + return nil, nil + } + return j, nil +} + +func (s *Memstore) GetNextEnqueuedJob(ctx context.Context) (Job, error) { + s.lk.RLock() + defer s.lk.RUnlock() + + for _, j := range s.jobs { + if j.State() == StateEnqueued { + return j, nil + } + } + return nil, nil +} + +func (s *Memstore) PurgeRepo(ctx context.Context, repo string) error { + s.lk.RLock() + defer s.lk.RUnlock() + + delete(s.jobs, repo) + return nil +} + +func (j *Memjob) Repo() string { + return j.repo +} + +func (j *Memjob) State() string { + j.lk.Lock() + defer j.lk.Unlock() + + return j.state +} + +func (j *Memjob) SetState(ctx context.Context, state string) error { + j.lk.Lock() + defer j.lk.Unlock() + + j.state = state + j.updatedAt = time.Now() + return nil +} + +func (j *Memjob) Rev() string { + return j.rev +} + +func (j *Memjob) SetRev(ctx context.Context, rev string) error { + j.rev = rev + return nil +} + +func (j *Memjob) FlushBufferedOps(ctx context.Context, fn func(kind repomgr.EventKind, rev, path string, rec *[]byte, cid *cid.Cid) error) error { + panic("TODO: copy what we end up doing from the gormstore") + /* + j.lk.Lock() + defer j.lk.Unlock() + + for _, opset := range j.bufferedOps { + for _, op := range opset.ops { + if err := fn(op.Kind, op.Path, op.Record, op.Cid); err != nil { + return err + } + } + } + + j.bufferedOps = map[string][]*BufferedOp{} + j.state = StateComplete + + return nil + */ +} + +func (j *Memjob) ClearBufferedOps(ctx context.Context) error { + j.lk.Lock() + defer j.lk.Unlock() + + j.bufferedOps = []*opSet{} + j.updatedAt = time.Now() + return nil +} + +func (j *Memjob) RetryCount() int { + j.lk.Lock() + defer j.lk.Unlock() + return 0 +} diff --git a/backfill/metrics.go b/backfill/metrics.go new file mode 100644 index 000000000..effa21ee2 --- /dev/null +++ b/backfill/metrics.go @@ -0,0 +1,31 @@ +package backfill + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var backfillJobsEnqueued = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "backfill_jobs_enqueued_total", + Help: "The total number of backfill jobs enqueued", +}, []string{"backfiller_name"}) + +var backfillJobsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "backfill_jobs_processed_total", + Help: "The total number of backfill jobs processed", +}, []string{"backfiller_name"}) + +var backfillRecordsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "backfill_records_processed_total", + Help: "The total number of backfill records processed", +}, []string{"backfiller_name"}) + +var backfillOpsBuffered = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "backfill_ops_buffered", + Help: "The number of backfill operations buffered", +}, []string{"backfiller_name"}) + +var backfillBytesProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "backfill_bytes_processed_total", + Help: "The total number of backfill bytes processed", +}, []string{"backfiller_name"}) diff --git a/backfill/util.go b/backfill/util.go new file mode 100644 index 000000000..f8f159290 --- /dev/null +++ b/backfill/util.go @@ -0,0 +1,33 @@ +package backfill + +import ( + "io" + + "github.com/prometheus/client_golang/prometheus" +) + +type instrumentedReader struct { + source io.ReadCloser + counter prometheus.Counter +} + +func (r instrumentedReader) Read(b []byte) (int, error) { + n, err := r.source.Read(b) + r.counter.Add(float64(n)) + return n, err +} + +func (r instrumentedReader) Close() error { + var buf [32]byte + var n int + var err error + for err == nil { + n, err = r.source.Read(buf[:]) + r.counter.Add(float64(n)) + } + closeerr := r.source.Close() + if err != nil && err != io.EOF { + return err + } + return closeerr +} diff --git a/bgs/admin.go b/bgs/admin.go new file mode 100644 index 000000000..d1f868d06 --- /dev/null +++ b/bgs/admin.go @@ -0,0 +1,724 @@ +package bgs + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "time" + + "github.com/bluesky-social/indigo/models" + "github.com/labstack/echo/v4" + dto "github.com/prometheus/client_model/go" + "go.opentelemetry.io/otel" + "golang.org/x/time/rate" + "gorm.io/gorm" +) + +func (bgs *BGS) handleAdminSetSubsEnabled(e echo.Context) error { + enabled, err := strconv.ParseBool(e.QueryParam("enabled")) + if err != nil { + return &echo.HTTPError{ + Code: 400, + Message: err.Error(), + } + } + + return bgs.slurper.SetNewSubsDisabled(!enabled) +} + +func (bgs *BGS) handleAdminGetSubsEnabled(e echo.Context) error { + return e.JSON(200, map[string]bool{ + "enabled": !bgs.slurper.GetNewSubsDisabledState(), + }) +} + +func (bgs *BGS) handleAdminGetNewPDSPerDayRateLimit(e echo.Context) error { + limit := bgs.slurper.GetNewPDSPerDayLimit() + return e.JSON(200, map[string]int64{ + "limit": limit, + }) +} + +func (bgs *BGS) handleAdminSetNewPDSPerDayRateLimit(e echo.Context) error { + limit, err := strconv.ParseInt(e.QueryParam("limit"), 10, 64) + if err != nil { + return &echo.HTTPError{ + Code: 400, + Message: fmt.Errorf("failed to parse limit: %w", err).Error(), + } + } + + err = bgs.slurper.SetNewPDSPerDayLimit(limit) + if err != nil { + return &echo.HTTPError{ + Code: 500, + Message: fmt.Errorf("failed to set new PDS per day rate limit: %w", err).Error(), + } + } + + return nil +} + +func (bgs *BGS) handleAdminTakeDownRepo(e echo.Context) error { + ctx := e.Request().Context() + + var body map[string]string + if err := e.Bind(&body); err != nil { + return err + } + did, ok := body["did"] + if !ok { + return &echo.HTTPError{ + Code: 400, + Message: "must specify did parameter in body", + } + } + + err := bgs.TakeDownRepo(ctx, did) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return &echo.HTTPError{ + Code: http.StatusNotFound, + Message: "repo not found", + } + } + return &echo.HTTPError{ + Code: http.StatusInternalServerError, + Message: err.Error(), + } + } + return nil +} + +func (bgs *BGS) handleAdminReverseTakedown(e echo.Context) error { + did := e.QueryParam("did") + ctx := e.Request().Context() + err := bgs.ReverseTakedown(ctx, did) + + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return &echo.HTTPError{ + Code: http.StatusNotFound, + Message: "repo not found", + } + } + return &echo.HTTPError{ + Code: http.StatusInternalServerError, + Message: err.Error(), + } + } + + return nil +} + +type ListTakedownsResponse struct { + Dids []string `json:"dids"` + Cursor int64 `json:"cursor,omitempty"` +} + +func (bgs *BGS) handleAdminListRepoTakeDowns(e echo.Context) error { + ctx := e.Request().Context() + haveMinId := false + minId := int64(-1) + qmin := e.QueryParam("cursor") + if qmin != "" { + tmin, err := strconv.ParseInt(qmin, 10, 64) + if err != nil { + return &echo.HTTPError{Code: 400, Message: "bad cursor"} + } + minId = tmin + haveMinId = true + } + limit := 1000 + wat := bgs.db.Model(User{}).WithContext(ctx).Select("id", "did").Where("taken_down = TRUE") + if haveMinId { + wat = wat.Where("id > ?", minId) + } + //var users []User + rows, err := wat.Order("id").Limit(limit).Rows() + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "oops").WithInternal(err) + } + var out ListTakedownsResponse + for rows.Next() { + var id int64 + var did string + err := rows.Scan(&id, &did) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "oops").WithInternal(err) + } + out.Dids = append(out.Dids, did) + out.Cursor = id + } + if len(out.Dids) < limit { + out.Cursor = 0 + } + return e.JSON(200, out) +} + +func (bgs *BGS) handleAdminGetUpstreamConns(e echo.Context) error { + return e.JSON(200, bgs.slurper.GetActiveList()) +} + +type rateLimit struct { + Max float64 `json:"Max"` + WindowSeconds float64 `json:"Window"` +} + +type enrichedPDS struct { + models.PDS + HasActiveConnection bool `json:"HasActiveConnection"` + EventsSeenSinceStartup uint64 `json:"EventsSeenSinceStartup"` + PerSecondEventRate rateLimit `json:"PerSecondEventRate"` + PerHourEventRate rateLimit `json:"PerHourEventRate"` + PerDayEventRate rateLimit `json:"PerDayEventRate"` + CrawlRate rateLimit `json:"CrawlRate"` + UserCount int64 `json:"UserCount"` +} + +type UserCount struct { + PDSID uint `gorm:"column:pds"` + UserCount int64 `gorm:"column:user_count"` +} + +func (bgs *BGS) handleListPDSs(e echo.Context) error { + var pds []models.PDS + if err := bgs.db.Find(&pds).Error; err != nil { + return err + } + + enrichedPDSs := make([]enrichedPDS, len(pds)) + + activePDSHosts := bgs.slurper.GetActiveList() + + for i, p := range pds { + enrichedPDSs[i].PDS = p + enrichedPDSs[i].HasActiveConnection = false + for _, host := range activePDSHosts { + if strings.ToLower(host) == strings.ToLower(p.Host) { + enrichedPDSs[i].HasActiveConnection = true + break + } + } + var m = &dto.Metric{} + if err := eventsReceivedCounter.WithLabelValues(p.Host).Write(m); err != nil { + enrichedPDSs[i].EventsSeenSinceStartup = 0 + continue + } + enrichedPDSs[i].EventsSeenSinceStartup = uint64(m.Counter.GetValue()) + + enrichedPDSs[i].PerSecondEventRate = rateLimit{ + Max: p.RateLimit, + WindowSeconds: 1, + } + + enrichedPDSs[i].PerHourEventRate = rateLimit{ + Max: float64(p.HourlyEventLimit), + WindowSeconds: 3600, + } + + enrichedPDSs[i].PerDayEventRate = rateLimit{ + Max: float64(p.DailyEventLimit), + WindowSeconds: 86400, + } + + // Get the crawl rate limit for this PDS + crawlRate := rateLimit{ + Max: p.CrawlRateLimit, + WindowSeconds: 1, + } + + enrichedPDSs[i].CrawlRate = crawlRate + } + + return e.JSON(200, enrichedPDSs) +} + +type consumer struct { + ID uint64 `json:"id"` + RemoteAddr string `json:"remote_addr"` + UserAgent string `json:"user_agent"` + EventsConsumed uint64 `json:"events_consumed"` + ConnectedAt time.Time `json:"connected_at"` +} + +func (bgs *BGS) handleAdminListConsumers(e echo.Context) error { + bgs.consumersLk.RLock() + defer bgs.consumersLk.RUnlock() + + consumers := make([]consumer, 0, len(bgs.consumers)) + for id, c := range bgs.consumers { + var m = &dto.Metric{} + if err := c.EventsSent.Write(m); err != nil { + continue + } + consumers = append(consumers, consumer{ + ID: id, + RemoteAddr: c.RemoteAddr, + UserAgent: c.UserAgent, + EventsConsumed: uint64(m.Counter.GetValue()), + ConnectedAt: c.ConnectedAt, + }) + } + + return e.JSON(200, consumers) +} + +func (bgs *BGS) handleAdminKillUpstreamConn(e echo.Context) error { + host := strings.TrimSpace(e.QueryParam("host")) + if host == "" { + return &echo.HTTPError{ + Code: 400, + Message: "must pass a valid host", + } + } + + block := strings.ToLower(e.QueryParam("block")) == "true" + + if err := bgs.slurper.KillUpstreamConnection(host, block); err != nil { + if errors.Is(err, ErrNoActiveConnection) { + return &echo.HTTPError{ + Code: 400, + Message: "no active connection to given host", + } + } + return err + } + + return e.JSON(200, map[string]any{ + "success": "true", + }) +} + +func (bgs *BGS) handleBlockPDS(e echo.Context) error { + host := strings.TrimSpace(e.QueryParam("host")) + if host == "" { + return &echo.HTTPError{ + Code: 400, + Message: "must pass a valid host", + } + } + + // Set the block flag to true in the DB + if err := bgs.db.Model(&models.PDS{}).Where("host = ?", host).Update("blocked", true).Error; err != nil { + return err + } + + // don't care if this errors, but we should try to disconnect something we just blocked + _ = bgs.slurper.KillUpstreamConnection(host, false) + + return e.JSON(200, map[string]any{ + "success": "true", + }) +} + +func (bgs *BGS) handleUnblockPDS(e echo.Context) error { + host := strings.TrimSpace(e.QueryParam("host")) + if host == "" { + return &echo.HTTPError{ + Code: 400, + Message: "must pass a valid host", + } + } + + // Set the block flag to false in the DB + if err := bgs.db.Model(&models.PDS{}).Where("host = ?", host).Update("blocked", false).Error; err != nil { + return err + } + + return e.JSON(200, map[string]any{ + "success": "true", + }) +} + +type bannedDomains struct { + BannedDomains []string `json:"banned_domains"` +} + +func (bgs *BGS) handleAdminListDomainBans(c echo.Context) error { + var all []models.DomainBan + if err := bgs.db.Find(&all).Error; err != nil { + return err + } + + resp := bannedDomains{ + BannedDomains: []string{}, + } + for _, b := range all { + resp.BannedDomains = append(resp.BannedDomains, b.Domain) + } + + return c.JSON(200, resp) +} + +type banDomainBody struct { + Domain string +} + +func (bgs *BGS) handleAdminBanDomain(c echo.Context) error { + var body banDomainBody + if err := c.Bind(&body); err != nil { + return err + } + + // Check if the domain is already banned + var existing models.DomainBan + if err := bgs.db.Where("domain = ?", body.Domain).First(&existing).Error; err == nil { + return &echo.HTTPError{ + Code: 400, + Message: "domain is already banned", + } + } + + if err := bgs.db.Create(&models.DomainBan{ + Domain: body.Domain, + }).Error; err != nil { + return err + } + + return c.JSON(200, map[string]any{ + "success": "true", + }) +} + +func (bgs *BGS) handleAdminUnbanDomain(c echo.Context) error { + var body banDomainBody + if err := c.Bind(&body); err != nil { + return err + } + + if err := bgs.db.Where("domain = ?", body.Domain).Delete(&models.DomainBan{}).Error; err != nil { + return err + } + + return c.JSON(200, map[string]any{ + "success": "true", + }) +} + +type PDSRates struct { + PerSecond int64 `json:"per_second,omitempty"` + PerHour int64 `json:"per_hour,omitempty"` + PerDay int64 `json:"per_day,omitempty"` + CrawlRate int64 `json:"crawl_rate,omitempty"` + RepoLimit int64 `json:"repo_limit,omitempty"` +} + +func (pr *PDSRates) FromSlurper(s *Slurper) { + if pr.PerSecond == 0 { + pr.PerHour = s.DefaultPerSecondLimit + } + if pr.PerHour == 0 { + pr.PerHour = s.DefaultPerHourLimit + } + if pr.PerDay == 0 { + pr.PerDay = s.DefaultPerDayLimit + } + if pr.CrawlRate == 0 { + pr.CrawlRate = int64(s.DefaultCrawlLimit) + } + if pr.RepoLimit == 0 { + pr.RepoLimit = s.DefaultRepoLimit + } +} + +type RateLimitChangeRequest struct { + Host string `json:"host"` + PDSRates +} + +func (bgs *BGS) handleAdminChangePDSRateLimits(e echo.Context) error { + var body RateLimitChangeRequest + if err := e.Bind(&body); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid body: %s", err)) + } + + // Get the PDS from the DB + var pds models.PDS + if err := bgs.db.Where("host = ?", body.Host).First(&pds).Error; err != nil { + return err + } + + // Update the rate limits in the DB + pds.RateLimit = float64(body.PerSecond) + pds.HourlyEventLimit = body.PerHour + pds.DailyEventLimit = body.PerDay + pds.CrawlRateLimit = float64(body.CrawlRate) + pds.RepoLimit = body.RepoLimit + + if err := bgs.db.Save(&pds).Error; err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Errorf("failed to save rate limit changes: %w", err)) + } + + // Update the rate limit in the limiter + limits := bgs.slurper.GetOrCreateLimiters(pds.ID, body.PerSecond, body.PerHour, body.PerDay) + limits.PerSecond.SetLimit(body.PerSecond) + limits.PerHour.SetLimit(body.PerHour) + limits.PerDay.SetLimit(body.PerDay) + + // Set the crawl rate limit + bgs.repoFetcher.GetOrCreateLimiter(pds.ID, float64(body.CrawlRate)).SetLimit(rate.Limit(body.CrawlRate)) + + return e.JSON(200, map[string]any{ + "success": "true", + }) +} + +func (bgs *BGS) handleAdminCompactRepo(e echo.Context) error { + ctx, span := otel.Tracer("bgs").Start(context.Background(), "adminCompactRepo") + defer span.End() + + did := e.QueryParam("did") + if did == "" { + return fmt.Errorf("must pass a did") + } + + var fast bool + if strings.ToLower(e.QueryParam("fast")) == "true" { + fast = true + } + + u, err := bgs.lookupUserByDid(ctx, did) + if err != nil { + return fmt.Errorf("no such user: %w", err) + } + + stats, err := bgs.repoman.CarStore().CompactUserShards(ctx, u.ID, fast) + if err != nil { + return fmt.Errorf("compaction failed: %w", err) + } + + return e.JSON(200, map[string]any{ + "success": "true", + "stats": stats, + }) +} + +func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { + ctx, span := otel.Tracer("bgs").Start(context.Background(), "adminCompactAllRepos") + defer span.End() + + var fast bool + if strings.ToLower(e.QueryParam("fast")) == "true" { + fast = true + } + + lim := 50 + if limstr := e.QueryParam("limit"); limstr != "" { + v, err := strconv.Atoi(limstr) + if err != nil { + return err + } + + lim = v + } + + shardThresh := 20 + if threshstr := e.QueryParam("threshold"); threshstr != "" { + v, err := strconv.Atoi(threshstr) + if err != nil { + return err + } + + shardThresh = v + } + + err := bgs.compactor.EnqueueAllRepos(ctx, bgs, lim, shardThresh, fast) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Errorf("failed to enqueue all repos: %w", err)) + } + + return e.JSON(200, map[string]any{ + "success": "true", + }) +} + +func (bgs *BGS) handleAdminPostResyncPDS(e echo.Context) error { + host := strings.TrimSpace(e.QueryParam("host")) + if host == "" { + return fmt.Errorf("must pass a host") + } + + // Get the PDS from the DB + var pds models.PDS + if err := bgs.db.Where("host = ?", host).First(&pds).Error; err != nil { + return err + } + + go func() { + ctx := context.Background() + err := bgs.ResyncPDS(ctx, pds) + if err != nil { + log.Error("failed to resync PDS", "err", err, "pds", pds.Host) + } + }() + + return e.JSON(200, map[string]any{ + "message": "resync started...", + }) +} + +func (bgs *BGS) handleAdminGetResyncPDS(e echo.Context) error { + host := strings.TrimSpace(e.QueryParam("host")) + if host == "" { + return fmt.Errorf("must pass a host") + } + + // Get the PDS from the DB + var pds models.PDS + if err := bgs.db.Where("host = ?", host).First(&pds).Error; err != nil { + return err + } + + resync, found := bgs.GetResync(pds) + if !found { + return &echo.HTTPError{ + Code: 404, + Message: "no resync found for given PDS", + } + } + + return e.JSON(200, map[string]any{ + "resync": resync, + }) +} + +func (bgs *BGS) handleAdminResetRepo(e echo.Context) error { + ctx := e.Request().Context() + + did := e.QueryParam("did") + if did == "" { + return fmt.Errorf("must pass a did") + } + + ai, err := bgs.Index.LookupUserByDid(ctx, did) + if err != nil { + return fmt.Errorf("no such user: %w", err) + } + + if err := bgs.repoman.ResetRepo(ctx, ai.Uid); err != nil { + return err + } + + if err := bgs.Index.Crawler.Crawl(ctx, ai); err != nil { + return err + } + + return e.JSON(200, map[string]any{ + "success": true, + }) +} + +func (bgs *BGS) handleAdminVerifyRepo(e echo.Context) error { + ctx := e.Request().Context() + + did := e.QueryParam("did") + if did == "" { + return fmt.Errorf("must pass a did") + } + + ai, err := bgs.Index.LookupUserByDid(ctx, did) + if err != nil { + return fmt.Errorf("no such user: %w", err) + } + + if err := bgs.repoman.VerifyRepo(ctx, ai.Uid); err != nil { + return err + } + + return e.JSON(200, map[string]any{ + "success": true, + }) +} + +func (bgs *BGS) handleAdminAddTrustedDomain(e echo.Context) error { + domain := e.QueryParam("domain") + if domain == "" { + return fmt.Errorf("must specify domain in query parameter") + } + + // Check if the domain is already trusted + trustedDomains := bgs.slurper.GetTrustedDomains() + if slices.Contains(trustedDomains, domain) { + return &echo.HTTPError{ + Code: 400, + Message: "domain is already trusted", + } + } + + if err := bgs.slurper.AddTrustedDomain(domain); err != nil { + return err + } + + return e.JSON(200, map[string]any{ + "success": true, + }) +} + +type AdminRequestCrawlRequest struct { + Hostname string `json:"hostname"` + + // optional: + PDSRates +} + +func (bgs *BGS) handleAdminRequestCrawl(e echo.Context) error { + ctx := e.Request().Context() + + var body AdminRequestCrawlRequest + if err := e.Bind(&body); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid body: %s", err)) + } + + host := body.Hostname + if host == "" { + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname") + } + + if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { + if bgs.ssl { + host = "https://" + host + } else { + host = "http://" + host + } + } + + u, err := url.Parse(host) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "failed to parse hostname") + } + + if u.Scheme == "http" && bgs.ssl { + return echo.NewHTTPError(http.StatusBadRequest, "this server requires https") + } + + if u.Scheme == "https" && !bgs.ssl { + return echo.NewHTTPError(http.StatusBadRequest, "this server does not support https") + } + + if u.Path != "" { + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without path") + } + + if u.Query().Encode() != "" { + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without query") + } + + host = u.Host // potentially hostname:port + + banned, err := bgs.domainIsBanned(ctx, host) + if banned { + return echo.NewHTTPError(http.StatusUnauthorized, "domain is banned") + } + + // Skip checking if the server is online for now + rateOverrides := body.PDSRates + rateOverrides.FromSlurper(bgs.slurper) + + return bgs.slurper.SubscribeToPds(ctx, host, true, true, &rateOverrides) // Override Trusted Domain Check +} diff --git a/bgs/bgs.go b/bgs/bgs.go index b8524933c..5ab61c5fb 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -2,121 +2,728 @@ package bgs import ( "context" + "database/sql" "errors" "fmt" + "log/slog" + "net" + "net/http" + _ "net/http/pprof" "net/url" + "strconv" "strings" + "sync" + "time" - bsky "github.com/bluesky-social/indigo/api/bsky" + atproto "github.com/bluesky-social/indigo/api/atproto" + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/carstore" + "github.com/bluesky-social/indigo/did" "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/handles" "github.com/bluesky-social/indigo/indexer" - "github.com/bluesky-social/indigo/plc" + "github.com/bluesky-social/indigo/models" "github.com/bluesky-social/indigo/repomgr" - "github.com/bluesky-social/indigo/types" + "github.com/bluesky-social/indigo/util/svcutil" "github.com/bluesky-social/indigo/xrpc" + lru "github.com/hashicorp/golang-lru/v2" + "golang.org/x/sync/semaphore" + "golang.org/x/time/rate" + "github.com/gorilla/websocket" + ipld "github.com/ipfs/go-ipld-format" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" - "github.com/labstack/gommon/log" + promclient "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + dto "github.com/prometheus/client_model/go" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "gorm.io/gorm" ) +var tracer = otel.Tracer("bgs") + +// serverListenerBootTimeout is how long to wait for the requested server socket +// to become available for use. This is an arbitrary timeout that should be safe +// on any platform, but there's no great way to weave this timeout without +// adding another parameter to the (at time of writing) long signature of +// NewServer. +const serverListenerBootTimeout = 5 * time.Second + type BGS struct { - index *indexer.Indexer - db *gorm.DB - slurper *Slurper - events *events.EventManager - didr plc.PLCClient + Index *indexer.Indexer + db *gorm.DB + slurper *Slurper + events *events.EventManager + didr did.Resolver + repoFetcher *indexer.RepoFetcher + + hr handles.HandleResolver + + // TODO: work on doing away with this flag in favor of more pluggable + // pieces that abstract the need for explicit ssl checks + ssl bool + + crawlOnly bool + + // TODO: at some point we will want to lock specific DIDs, this lock as is + // is overly broad, but i dont expect it to be a bottleneck for now + extUserLk sync.Mutex repoman *repomgr.RepoManager + + // Management of Socket Consumers + consumersLk sync.RWMutex + nextConsumerID uint64 + consumers map[uint64]*SocketConsumer + + // Management of Resyncs + pdsResyncsLk sync.RWMutex + pdsResyncs map[uint]*PDSResync + + // Management of Compaction + compactor *Compactor + + // User cache + userCache *lru.Cache[string, *User] + + // nextCrawlers gets forwarded POST /xrpc/com.atproto.sync.requestCrawl + nextCrawlers []*url.URL + httpClient http.Client + + log *slog.Logger +} + +type PDSResync struct { + PDS models.PDS `json:"pds"` + NumRepoPages int `json:"numRepoPages"` + NumRepos int `json:"numRepos"` + NumReposChecked int `json:"numReposChecked"` + NumReposToResync int `json:"numReposToResync"` + Status string `json:"status"` + StatusChangedAt time.Time `json:"statusChangedAt"` } -func NewBGS(db *gorm.DB, ix *indexer.Indexer, repoman *repomgr.RepoManager, evtman *events.EventManager, didr plc.PLCClient) *BGS { +type SocketConsumer struct { + UserAgent string + RemoteAddr string + ConnectedAt time.Time + EventsSent promclient.Counter +} + +type BGSConfig struct { + SSL bool + CompactInterval time.Duration + DefaultRepoLimit int64 + ConcurrencyPerPDS int64 + MaxQueuePerPDS int64 + NumCompactionWorkers int + + // NextCrawlers gets forwarded POST /xrpc/com.atproto.sync.requestCrawl + NextCrawlers []*url.URL +} + +func DefaultBGSConfig() *BGSConfig { + return &BGSConfig{ + SSL: true, + CompactInterval: 4 * time.Hour, + DefaultRepoLimit: 100, + ConcurrencyPerPDS: 100, + MaxQueuePerPDS: 1_000, + NumCompactionWorkers: 2, + } +} + +func NewBGS(db *gorm.DB, ix *indexer.Indexer, repoman *repomgr.RepoManager, evtman *events.EventManager, didr did.Resolver, rf *indexer.RepoFetcher, hr handles.HandleResolver, config *BGSConfig) (*BGS, error) { + + if config == nil { + config = DefaultBGSConfig() + } db.AutoMigrate(User{}) - db.AutoMigrate(PDS{}) + db.AutoMigrate(AuthToken{}) + db.AutoMigrate(models.PDS{}) + db.AutoMigrate(models.DomainBan{}) + + uc, _ := lru.New[string, *User](1_000_000) bgs := &BGS{ - index: ix, - db: db, + Index: ix, + db: db, + repoFetcher: rf, + hr: hr, repoman: repoman, events: evtman, didr: didr, + ssl: config.SSL, + + consumersLk: sync.RWMutex{}, + consumers: make(map[uint64]*SocketConsumer), + + pdsResyncs: make(map[uint]*PDSResync), + + userCache: uc, + + log: slog.Default().With("system", "bgs"), } - bgs.slurper = NewSlurper(db, bgs.handleFedEvent) - return bgs + + ix.CreateExternalUser = bgs.createExternalUser + slOpts := DefaultSlurperOptions() + slOpts.SSL = config.SSL + slOpts.DefaultRepoLimit = config.DefaultRepoLimit + slOpts.ConcurrencyPerPDS = config.ConcurrencyPerPDS + slOpts.MaxQueuePerPDS = config.MaxQueuePerPDS + s, err := NewSlurper(db, bgs.handleFedEvent, slOpts) + if err != nil { + return nil, err + } + + bgs.slurper = s + + if err := bgs.slurper.RestartAll(); err != nil { + return nil, err + } + + cOpts := DefaultCompactorOptions() + cOpts.NumWorkers = config.NumCompactionWorkers + compactor := NewCompactor(cOpts) + compactor.requeueInterval = config.CompactInterval + compactor.Start(bgs) + bgs.compactor = compactor + + bgs.nextCrawlers = config.NextCrawlers + bgs.httpClient.Timeout = time.Second * 5 + + return bgs, nil +} + +func (bgs *BGS) StartMetrics(listen string) error { + http.Handle("/metrics", promhttp.Handler()) + return http.ListenAndServe(listen, nil) +} + +func (bgs *BGS) Start(addr string) error { + var lc net.ListenConfig + ctx, cancel := context.WithTimeout(context.Background(), serverListenerBootTimeout) + defer cancel() + + li, err := lc.Listen(ctx, "tcp", addr) + if err != nil { + return err + } + return bgs.StartWithListener(li) } -func (bgs *BGS) Start(listen string) error { +func (bgs *BGS) StartWithListener(listen net.Listener) error { e := echo.New() + e.HideBanner = true - e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ - Format: "method=${method}, uri=${uri}, status=${status} latency=${latency_human}\n", + e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ + AllowOrigins: []string{"*"}, + AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderContentType, echo.HeaderAccept, echo.HeaderAuthorization}, })) + if !bgs.ssl { + e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ + Format: "method=${method}, uri=${uri}, status=${status} latency=${latency_human}\n", + })) + } else { + e.Use(middleware.LoggerWithConfig(middleware.DefaultLoggerConfig)) + } + + // React uses a virtual router, so we need to serve the index.html for all + // routes that aren't otherwise handled or in the /assets directory. + e.File("/dash", "public/index.html") + e.File("/dash/*", "public/index.html") + e.Static("/assets", "public/assets") + + e.Use(svcutil.MetricsMiddleware) + e.HTTPErrorHandler = func(err error, ctx echo.Context) { - fmt.Printf("HANDLER ERROR: (%s) %s\n", ctx.Path(), err) - ctx.Response().WriteHeader(500) + switch err := err.(type) { + case *echo.HTTPError: + if err2 := ctx.JSON(err.Code, map[string]any{ + "error": err.Message, + }); err2 != nil { + bgs.log.Error("Failed to write http error", "err", err2) + } + default: + sendHeader := true + if ctx.Path() == "/xrpc/com.atproto.sync.subscribeRepos" { + sendHeader = false + } + + bgs.log.Warn("HANDLER ERROR: (%s) %s", ctx.Path(), err) + + if strings.HasPrefix(ctx.Path(), "/admin/") { + ctx.JSON(500, map[string]any{ + "error": err.Error(), + }) + return + } + + if sendHeader { + ctx.Response().WriteHeader(500) + } + } } // TODO: this API is temporary until we formalize what we want here - e.POST("/add-target", bgs.handleAddTarget) - e.GET("/events", bgs.EventsHandler) + e.GET("/xrpc/com.atproto.sync.subscribeRepos", bgs.EventsHandler) + e.GET("/xrpc/com.atproto.sync.getRecord", bgs.HandleComAtprotoSyncGetRecord) + e.GET("/xrpc/com.atproto.sync.getRepo", bgs.HandleComAtprotoSyncGetRepo) + e.GET("/xrpc/com.atproto.sync.getBlocks", bgs.HandleComAtprotoSyncGetBlocks) + e.GET("/xrpc/com.atproto.sync.requestCrawl", bgs.HandleComAtprotoSyncRequestCrawl) + e.POST("/xrpc/com.atproto.sync.requestCrawl", bgs.HandleComAtprotoSyncRequestCrawl) + e.GET("/xrpc/com.atproto.sync.listRepos", bgs.HandleComAtprotoSyncListRepos) + e.GET("/xrpc/com.atproto.sync.getLatestCommit", bgs.HandleComAtprotoSyncGetLatestCommit) + e.GET("/xrpc/com.atproto.sync.notifyOfUpdate", bgs.HandleComAtprotoSyncNotifyOfUpdate) + e.GET("/xrpc/_health", bgs.HandleHealthCheck) + e.GET("/_health", bgs.HandleHealthCheck) + e.GET("/", bgs.HandleHomeMessage) + + admin := e.Group("/admin", bgs.checkAdminAuth) + + // Slurper-related Admin API + admin.GET("/subs/getUpstreamConns", bgs.handleAdminGetUpstreamConns) + admin.GET("/subs/getEnabled", bgs.handleAdminGetSubsEnabled) + admin.GET("/subs/perDayLimit", bgs.handleAdminGetNewPDSPerDayRateLimit) + admin.POST("/subs/setEnabled", bgs.handleAdminSetSubsEnabled) + admin.POST("/subs/killUpstream", bgs.handleAdminKillUpstreamConn) + admin.POST("/subs/setPerDayLimit", bgs.handleAdminSetNewPDSPerDayRateLimit) + + // Domain-related Admin API + admin.GET("/subs/listDomainBans", bgs.handleAdminListDomainBans) + admin.POST("/subs/banDomain", bgs.handleAdminBanDomain) + admin.POST("/subs/unbanDomain", bgs.handleAdminUnbanDomain) + + // Repo-related Admin API + admin.POST("/repo/takeDown", bgs.handleAdminTakeDownRepo) + admin.POST("/repo/reverseTakedown", bgs.handleAdminReverseTakedown) + admin.GET("/repo/takedowns", bgs.handleAdminListRepoTakeDowns) + admin.POST("/repo/compact", bgs.handleAdminCompactRepo) + admin.POST("/repo/compactAll", bgs.handleAdminCompactAllRepos) + admin.POST("/repo/reset", bgs.handleAdminResetRepo) + admin.POST("/repo/verify", bgs.handleAdminVerifyRepo) + + // PDS-related Admin API + admin.POST("/pds/requestCrawl", bgs.handleAdminRequestCrawl) + admin.GET("/pds/list", bgs.handleListPDSs) + admin.POST("/pds/resync", bgs.handleAdminPostResyncPDS) + admin.GET("/pds/resync", bgs.handleAdminGetResyncPDS) + admin.POST("/pds/changeLimits", bgs.handleAdminChangePDSRateLimits) + admin.POST("/pds/block", bgs.handleBlockPDS) + admin.POST("/pds/unblock", bgs.handleUnblockPDS) + admin.POST("/pds/addTrustedDomain", bgs.handleAdminAddTrustedDomain) + + // Consumer-related Admin API + admin.GET("/consumers/list", bgs.handleAdminListConsumers) + + // In order to support booting on random ports in tests, we need to tell the + // Echo instance it's already got a port, and then use its StartServer + // method to re-use that listener. + e.Listener = listen + srv := &http.Server{} + return e.StartServer(srv) +} + +func (bgs *BGS) Shutdown() []error { + errs := bgs.slurper.Shutdown() + + if err := bgs.events.Shutdown(context.TODO()); err != nil { + errs = append(errs, err) + } + + bgs.compactor.Shutdown() + + return errs +} + +type HealthStatus struct { + Status string `json:"status"` + Message string `json:"msg,omitempty"` +} - return e.Start(listen) +func (bgs *BGS) HandleHealthCheck(c echo.Context) error { + if err := bgs.db.Exec("SELECT 1").Error; err != nil { + bgs.log.Error("healthcheck can't connect to database", "err", err) + return c.JSON(500, HealthStatus{Status: "error", Message: "can't connect to database"}) + } else { + return c.JSON(200, HealthStatus{Status: "ok"}) + } } -type PDS struct { +var homeMessage string = ` +d8888b. d888888b d888b .d8888. db dD db db +88 '8D '88' 88' Y8b 88' YP 88 ,8P' '8b d8' +88oooY' 88 88 '8bo. 88,8P '8bd8' +88~~~b. 88 88 ooo 'Y8b. 88'8b 88 +88 8D .88. 88. ~8~ db 8D 88 '88. 88 +Y8888P' Y888888P Y888P '8888Y' YP YD YP + +This is an atproto [https://atproto.com] relay instance, running the 'bigsky' codebase [https://github.com/bluesky-social/indigo] + +The firehose WebSocket path is at: /xrpc/com.atproto.sync.subscribeRepos +` + +func (bgs *BGS) HandleHomeMessage(c echo.Context) error { + return c.String(http.StatusOK, homeMessage) +} + +type AuthToken struct { gorm.Model + Token string `gorm:"index"` +} + +func (bgs *BGS) lookupAdminToken(tok string) (bool, error) { + var at AuthToken + if err := bgs.db.Find(&at, "token = ?", tok).Error; err != nil { + return false, err + } - Host string + if at.ID == 0 { + return false, nil + } + + return true, nil +} + +func (bgs *BGS) CreateAdminToken(tok string) error { + exists, err := bgs.lookupAdminToken(tok) + if err != nil { + return err + } + + if exists { + return nil + } + + return bgs.db.Create(&AuthToken{ + Token: tok, + }).Error +} + +func (bgs *BGS) checkAdminAuth(next echo.HandlerFunc) echo.HandlerFunc { + return func(e echo.Context) error { + ctx, span := tracer.Start(e.Request().Context(), "checkAdminAuth") + defer span.End() + + e.SetRequest(e.Request().WithContext(ctx)) + + authheader := e.Request().Header.Get("Authorization") + pref := "Bearer " + if !strings.HasPrefix(authheader, pref) { + return echo.ErrForbidden + } + + token := authheader[len(pref):] + + exists, err := bgs.lookupAdminToken(token) + if err != nil { + return err + } + + if !exists { + return echo.ErrForbidden + } + + return next(e) + } } type User struct { - gorm.Model - Handle string `gorm:"uniqueIndex"` - Did string `gorm:"uniqueIndex"` - PDS uint + ID models.Uid `gorm:"primarykey;index:idx_user_id_active,where:taken_down = false AND tombstoned = false"` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt gorm.DeletedAt `gorm:"index"` + Handle sql.NullString `gorm:"index"` + Did string `gorm:"uniqueIndex"` + PDS uint + ValidHandle bool `gorm:"default:true"` + + // TakenDown is set to true if the user in question has been taken down. + // A user in this state will have all future events related to it dropped + // and no data about this user will be served. + TakenDown bool + Tombstoned bool + + // UpstreamStatus is the state of the user as reported by the upstream PDS + UpstreamStatus string `gorm:"index"` + + lk sync.Mutex +} + +func (u *User) SetTakenDown(v bool) { + u.lk.Lock() + defer u.lk.Unlock() + u.TakenDown = v +} + +func (u *User) GetTakenDown() bool { + u.lk.Lock() + defer u.lk.Unlock() + return u.TakenDown +} + +func (u *User) SetTombstoned(v bool) { + u.lk.Lock() + defer u.lk.Unlock() + u.Tombstoned = v +} + +func (u *User) GetTombstoned() bool { + u.lk.Lock() + defer u.lk.Unlock() + return u.Tombstoned +} + +func (u *User) SetUpstreamStatus(v string) { + u.lk.Lock() + defer u.lk.Unlock() + u.UpstreamStatus = v +} + +func (u *User) GetUpstreamStatus() string { + u.lk.Lock() + defer u.lk.Unlock() + return u.UpstreamStatus } type addTargetBody struct { - Host string + Host string `json:"host"` } -// the ding-dong api -func (bgs *BGS) handleAddTarget(c echo.Context) error { - var body addTargetBody - if err := c.Bind(&body); err != nil { - return err +func (bgs *BGS) registerConsumer(c *SocketConsumer) uint64 { + bgs.consumersLk.Lock() + defer bgs.consumersLk.Unlock() + + id := bgs.nextConsumerID + bgs.nextConsumerID++ + + bgs.consumers[id] = c + + return id +} + +func (bgs *BGS) cleanupConsumer(id uint64) { + bgs.consumersLk.Lock() + defer bgs.consumersLk.Unlock() + + c := bgs.consumers[id] + + var m = &dto.Metric{} + if err := c.EventsSent.Write(m); err != nil { + bgs.log.Error("failed to get sent counter", "err", err) } - return bgs.slurper.SubscribeToPds(c.Request().Context(), body.Host) + bgs.log.Info("consumer disconnected", + "consumer_id", id, + "remote_addr", c.RemoteAddr, + "user_agent", c.UserAgent, + "events_sent", m.Counter.GetValue()) + + delete(bgs.consumers, id) } func (bgs *BGS) EventsHandler(c echo.Context) error { + var since *int64 + if sinceVal := c.QueryParam("cursor"); sinceVal != "" { + sval, err := strconv.ParseInt(sinceVal, 10, 64) + if err != nil { + return err + } + since = &sval + } + + ctx, cancel := context.WithCancel(c.Request().Context()) + defer cancel() + // TODO: authhhh - conn, err := websocket.Upgrade(c.Response().Writer, c.Request(), c.Response().Header(), 1<<10, 1<<10) + conn, err := websocket.Upgrade(c.Response(), c.Request(), c.Response().Header(), 10<<10, 10<<10) if err != nil { - return err + return fmt.Errorf("upgrading websocket: %w", err) } - evts, cancel, err := bgs.events.Subscribe(func(evt *events.Event) bool { return true }) + defer conn.Close() + + lastWriteLk := sync.Mutex{} + lastWrite := time.Now() + + // Start a goroutine to ping the client every 30 seconds to check if it's + // still alive. If the client doesn't respond to a ping within 5 seconds, + // we'll close the connection and teardown the consumer. + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + lastWriteLk.Lock() + lw := lastWrite + lastWriteLk.Unlock() + + if time.Since(lw) < 30*time.Second { + continue + } + + if err := conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(5*time.Second)); err != nil { + bgs.log.Warn("failed to ping client", "err", err) + cancel() + return + } + case <-ctx.Done(): + return + } + } + }() + + conn.SetPingHandler(func(message string) error { + err := conn.WriteControl(websocket.PongMessage, []byte(message), time.Now().Add(time.Second*60)) + if err == websocket.ErrCloseSent { + return nil + } else if e, ok := err.(net.Error); ok && e.Temporary() { + return nil + } + return err + }) + + // Start a goroutine to read messages from the client and discard them. + go func() { + for { + _, _, err := conn.ReadMessage() + if err != nil { + bgs.log.Warn("failed to read message from client", "err", err) + cancel() + return + } + } + }() + + ident := c.RealIP() + "-" + c.Request().UserAgent() + + evts, cleanup, err := bgs.events.Subscribe(ctx, ident, func(evt *events.XRPCStreamEvent) bool { return true }, since) if err != nil { return err } - defer cancel() + defer cleanup() - for evt := range evts { - if err := conn.WriteJSON(evt); err != nil { - return err + // Keep track of the consumer for metrics and admin endpoints + consumer := SocketConsumer{ + RemoteAddr: c.RealIP(), + UserAgent: c.Request().UserAgent(), + ConnectedAt: time.Now(), + } + sentCounter := eventsSentCounter.WithLabelValues(consumer.RemoteAddr, consumer.UserAgent) + consumer.EventsSent = sentCounter + + consumerID := bgs.registerConsumer(&consumer) + defer bgs.cleanupConsumer(consumerID) + + logger := bgs.log.With( + "consumer_id", consumerID, + "remote_addr", consumer.RemoteAddr, + "user_agent", consumer.UserAgent, + ) + + logger.Info("new consumer", "cursor", since) + + for { + select { + case evt, ok := <-evts: + if !ok { + logger.Error("event stream closed unexpectedly") + return nil + } + + wc, err := conn.NextWriter(websocket.BinaryMessage) + if err != nil { + logger.Error("failed to get next writer", "err", err) + return err + } + + if evt.Preserialized != nil { + _, err = wc.Write(evt.Preserialized) + } else { + err = evt.Serialize(wc) + } + if err != nil { + return fmt.Errorf("failed to write event: %w", err) + } + + if err := wc.Close(); err != nil { + logger.Warn("failed to flush-close our event write", "err", err) + return nil + } + + lastWriteLk.Lock() + lastWrite = time.Now() + lastWriteLk.Unlock() + sentCounter.Inc() + case <-ctx.Done(): + return nil } } +} - return nil +// domainIsBanned checks if the given host is banned, starting with the host +// itself, then checking every parent domain up to the tld +func (s *BGS) domainIsBanned(ctx context.Context, host string) (bool, error) { + // ignore ports when checking for ban status + hostport := strings.Split(host, ":") + + segments := strings.Split(hostport[0], ".") + + // TODO: use normalize method once that merges + var cleaned []string + for _, s := range segments { + if s == "" { + continue + } + s = strings.ToLower(s) + + cleaned = append(cleaned, s) + } + segments = cleaned + + for i := 0; i < len(segments)-1; i++ { + dchk := strings.Join(segments[i:], ".") + found, err := s.findDomainBan(ctx, dchk) + if err != nil { + return false, err + } + + if found { + return true, nil + } + } + return false, nil +} + +func (s *BGS) findDomainBan(ctx context.Context, host string) (bool, error) { + var db models.DomainBan + if err := s.db.Find(&db, "domain = ?", host).Error; err != nil { + return false, err + } + + if db.ID == 0 { + return false, nil + } + + return true, nil } func (bgs *BGS) lookupUserByDid(ctx context.Context, did string) (*User, error) { + ctx, span := tracer.Start(ctx, "lookupUserByDid") + defer span.End() + + cu, ok := bgs.userCache.Get(did) + if ok { + return cu, nil + } + var u User if err := bgs.db.Find(&u, "did = ?", did).Error; err != nil { return nil, err @@ -126,39 +733,293 @@ func (bgs *BGS) lookupUserByDid(ctx context.Context, did string) (*User, error) return nil, gorm.ErrRecordNotFound } + bgs.userCache.Add(did, &u) + return &u, nil } -func (bgs *BGS) handleFedEvent(ctx context.Context, host *PDS, evt *events.Event) error { - log.Infof("got fed event from %q: %s\n", host.Host, evt.Kind) - switch evt.Kind { - case events.EvtKindRepoChange: +func (bgs *BGS) lookupUserByUID(ctx context.Context, uid models.Uid) (*User, error) { + ctx, span := tracer.Start(ctx, "lookupUserByUID") + defer span.End() + + var u User + if err := bgs.db.Find(&u, "id = ?", uid).Error; err != nil { + return nil, err + } + + if u.ID == 0 { + return nil, gorm.ErrRecordNotFound + } + + return &u, nil +} + +func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *events.XRPCStreamEvent) error { + ctx, span := tracer.Start(ctx, "handleFedEvent") + defer span.End() + + start := time.Now() + defer func() { + eventsHandleDuration.WithLabelValues(host.Host).Observe(time.Since(start).Seconds()) + }() + + eventsReceivedCounter.WithLabelValues(host.Host).Add(1) + + switch { + case env.RepoCommit != nil: + repoCommitsReceivedCounter.WithLabelValues(host.Host).Add(1) + evt := env.RepoCommit + bgs.log.Debug("bgs got repo append event", "seq", evt.Seq, "pdsHost", host.Host, "repo", evt.Repo) + + s := time.Now() u, err := bgs.lookupUserByDid(ctx, evt.Repo) + userLookupDuration.Observe(time.Since(s).Seconds()) if err != nil { if !errors.Is(err, gorm.ErrRecordNotFound) { + repoCommitsResultCounter.WithLabelValues(host.Host, "nou").Inc() return fmt.Errorf("looking up event user: %w", err) } + newUsersDiscovered.Inc() + start := time.Now() subj, err := bgs.createExternalUser(ctx, evt.Repo) + newUserDiscoveryDuration.Observe(time.Since(start).Seconds()) if err != nil { - return err + repoCommitsResultCounter.WithLabelValues(host.Host, "uerr").Inc() + return fmt.Errorf("fed event create external user: %w", err) } u = new(User) u.ID = subj.Uid + u.Did = evt.Repo + } + + ustatus := u.GetUpstreamStatus() + span.SetAttributes(attribute.String("upstream_status", ustatus)) + + if u.GetTakenDown() || ustatus == events.AccountStatusTakendown { + span.SetAttributes(attribute.Bool("taken_down_by_relay_admin", u.GetTakenDown())) + bgs.log.Debug("dropping commit event from taken down user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host) + repoCommitsResultCounter.WithLabelValues(host.Host, "tdu").Inc() + return nil + } + + if ustatus == events.AccountStatusSuspended { + bgs.log.Debug("dropping commit event from suspended user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host) + repoCommitsResultCounter.WithLabelValues(host.Host, "susu").Inc() + return nil } - return bgs.repoman.HandleExternalUserEvent(ctx, host.ID, u.ID, evt.RepoOps, evt.CarSlice) + if ustatus == events.AccountStatusDeactivated { + bgs.log.Debug("dropping commit event from deactivated user", "did", evt.Repo, "seq", evt.Seq, "pdsHost", host.Host) + repoCommitsResultCounter.WithLabelValues(host.Host, "du").Inc() + return nil + } + + if evt.Rebase { + repoCommitsResultCounter.WithLabelValues(host.Host, "rebase").Inc() + return fmt.Errorf("rebase was true in event seq:%d,host:%s", evt.Seq, host.Host) + } + + if host.ID != u.PDS && u.PDS != 0 { + bgs.log.Warn("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", u.PDS, "gotPds", host.Host) + // Flush any cached DID documents for this user + bgs.didr.FlushCacheFor(env.RepoCommit.Repo) + + subj, err := bgs.createExternalUser(ctx, evt.Repo) + if err != nil { + repoCommitsResultCounter.WithLabelValues(host.Host, "uerr2").Inc() + return err + } + + if subj.PDS != host.ID { + repoCommitsResultCounter.WithLabelValues(host.Host, "noauth").Inc() + return fmt.Errorf("event from non-authoritative pds") + } + } + + if u.GetTombstoned() { + span.SetAttributes(attribute.Bool("tombstoned", true)) + // we've checked the authority of the users PDS, so reinstate the account + if err := bgs.db.Model(&User{}).Where("id = ?", u.ID).UpdateColumn("tombstoned", false).Error; err != nil { + repoCommitsResultCounter.WithLabelValues(host.Host, "tomb").Inc() + return fmt.Errorf("failed to un-tombstone a user: %w", err) + } + u.SetTombstoned(false) + + ai, err := bgs.Index.LookupUser(ctx, u.ID) + if err != nil { + repoCommitsResultCounter.WithLabelValues(host.Host, "nou2").Inc() + return fmt.Errorf("failed to look up user (tombstone recover): %w", err) + } + + // Now a simple re-crawl should suffice to bring the user back online + repoCommitsResultCounter.WithLabelValues(host.Host, "catchupt").Inc() + return bgs.Index.Crawler.AddToCatchupQueue(ctx, host, ai, evt) + } + + // skip the fast path for rebases or if the user is already in the slow path + if bgs.Index.Crawler.RepoInSlowPath(ctx, u.ID) { + rebasesCounter.WithLabelValues(host.Host).Add(1) + ai, err := bgs.Index.LookupUser(ctx, u.ID) + if err != nil { + repoCommitsResultCounter.WithLabelValues(host.Host, "nou3").Inc() + return fmt.Errorf("failed to look up user (slow path): %w", err) + } + + // TODO: we currently do not handle events that get queued up + // behind an already 'in progress' slow path event. + // this is strictly less efficient than it could be, and while it + // does 'work' (due to falling back to resyncing the repo), its + // technically incorrect. Now that we have the parallel event + // processor coming off of the pds stream, we should investigate + // whether or not we even need this 'slow path' logic, as it makes + // accounting for which events have been processed much harder + repoCommitsResultCounter.WithLabelValues(host.Host, "catchup").Inc() + return bgs.Index.Crawler.AddToCatchupQueue(ctx, host, ai, evt) + } + + if err := bgs.repoman.HandleExternalUserEvent(ctx, host.ID, u.ID, u.Did, evt.Since, evt.Rev, evt.Blocks, evt.Ops); err != nil { + + if errors.Is(err, carstore.ErrRepoBaseMismatch) || ipld.IsNotFound(err) { + ai, lerr := bgs.Index.LookupUser(ctx, u.ID) + if lerr != nil { + log.Warn("failed handling event, no user", "err", err, "pdsHost", host.Host, "seq", evt.Seq, "repo", u.Did, "commit", evt.Commit.String()) + repoCommitsResultCounter.WithLabelValues(host.Host, "nou4").Inc() + return fmt.Errorf("failed to look up user %s (%d) (err case: %s): %w", u.Did, u.ID, err, lerr) + } + + span.SetAttributes(attribute.Bool("catchup_queue", true)) + + log.Info("failed handling event, catchup", "err", err, "pdsHost", host.Host, "seq", evt.Seq, "repo", u.Did, "commit", evt.Commit.String()) + repoCommitsResultCounter.WithLabelValues(host.Host, "catchup2").Inc() + return bgs.Index.Crawler.AddToCatchupQueue(ctx, host, ai, evt) + } + + log.Warn("failed handling event", "err", err, "pdsHost", host.Host, "seq", evt.Seq, "repo", u.Did, "commit", evt.Commit.String()) + repoCommitsResultCounter.WithLabelValues(host.Host, "err").Inc() + return fmt.Errorf("handle user event failed: %w", err) + } + + repoCommitsResultCounter.WithLabelValues(host.Host, "ok").Inc() + return nil + case env.RepoIdentity != nil: + bgs.log.Info("bgs got identity event", "did", env.RepoIdentity.Did) + // Flush any cached DID documents for this user + bgs.didr.FlushCacheFor(env.RepoIdentity.Did) + + // Refetch the DID doc and update our cached keys and handle etc. + _, err := bgs.createExternalUser(ctx, env.RepoIdentity.Did) + if err != nil { + return err + } + + // Broadcast the identity event to all consumers + err = bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{ + RepoIdentity: &comatproto.SyncSubscribeRepos_Identity{ + Did: env.RepoIdentity.Did, + Seq: env.RepoIdentity.Seq, + Time: env.RepoIdentity.Time, + Handle: env.RepoIdentity.Handle, + }, + }) + if err != nil { + bgs.log.Error("failed to broadcast Identity event", "error", err, "did", env.RepoIdentity.Did) + return fmt.Errorf("failed to broadcast Identity event: %w", err) + } + + return nil + case env.RepoAccount != nil: + span.SetAttributes( + attribute.String("did", env.RepoAccount.Did), + attribute.Int64("seq", env.RepoAccount.Seq), + attribute.Bool("active", env.RepoAccount.Active), + ) + + if env.RepoAccount.Status != nil { + span.SetAttributes(attribute.String("repo_status", *env.RepoAccount.Status)) + } + + bgs.log.Info("bgs got account event", "did", env.RepoAccount.Did) + // Flush any cached DID documents for this user + bgs.didr.FlushCacheFor(env.RepoAccount.Did) + + // Refetch the DID doc to make sure the PDS is still authoritative + ai, err := bgs.createExternalUser(ctx, env.RepoAccount.Did) + if err != nil { + span.RecordError(err) + return err + } + + // Check if the PDS is still authoritative + // if not we don't want to be propagating this account event + if ai.PDS != host.ID { + bgs.log.Error("account event from non-authoritative pds", + "seq", env.RepoAccount.Seq, + "did", env.RepoAccount.Did, + "event_from", host.Host, + "did_doc_declared_pds", ai.PDS, + "account_evt", env.RepoAccount, + ) + return fmt.Errorf("event from non-authoritative pds") + } + + // Process the account status change + repoStatus := events.AccountStatusActive + if !env.RepoAccount.Active && env.RepoAccount.Status != nil { + repoStatus = *env.RepoAccount.Status + } + + err = bgs.UpdateAccountStatus(ctx, env.RepoAccount.Did, repoStatus) + if err != nil { + span.RecordError(err) + return fmt.Errorf("failed to update account status: %w", err) + } + + shouldBeActive := env.RepoAccount.Active + status := env.RepoAccount.Status + u, err := bgs.lookupUserByDid(ctx, env.RepoAccount.Did) + if err != nil { + return fmt.Errorf("failed to look up user by did: %w", err) + } + + if u.GetTakenDown() { + shouldBeActive = false + status = &events.AccountStatusTakendown + } + + // Broadcast the account event to all consumers + err = bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{ + RepoAccount: &comatproto.SyncSubscribeRepos_Account{ + Did: env.RepoAccount.Did, + Seq: env.RepoAccount.Seq, + Time: env.RepoAccount.Time, + Active: shouldBeActive, + Status: status, + }, + }) + if err != nil { + bgs.log.Error("failed to broadcast Account event", "error", err, "did", env.RepoAccount.Did) + return fmt.Errorf("failed to broadcast Account event: %w", err) + } + + return nil default: - return fmt.Errorf("unrecognized fed event kind: %q", evt.Kind) + return fmt.Errorf("invalid fed event") } - return nil } -func (s *BGS) createExternalUser(ctx context.Context, did string) (*types.ActorInfo, error) { +// TODO: rename? This also updates users, and 'external' is an old phrasing +func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.ActorInfo, error) { + ctx, span := tracer.Start(ctx, "createExternalUser") + defer span.End() + + externalUserCreationAttempts.Inc() + + s.log.Debug("create external user", "did", did) doc, err := s.didr.GetDocument(ctx, did) if err != nil { - return nil, fmt.Errorf("could not locate DID document for followed user: %s", err) + return nil, fmt.Errorf("could not locate DID document for followed user (%s): %w", did, err) } if len(doc.Service) == 0 { @@ -176,56 +1037,504 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*types.ActorI } // TODO: the PDS's DID should also be in the service, we could use that to look up? - var peering PDS - if err := s.db.First(&peering, "host = ?", durl.Host).Error; err != nil { + var peering models.PDS + if err := s.db.Find(&peering, "host = ?", durl.Host).Error; err != nil { + s.log.Error("failed to find pds", "host", durl.Host) return nil, err } - var handle string - if len(doc.AlsoKnownAs) > 0 { - hurl, err := url.Parse(doc.AlsoKnownAs[0]) + ban, err := s.domainIsBanned(ctx, durl.Host) + if err != nil { + return nil, fmt.Errorf("failed to check pds ban status: %w", err) + } + + if ban { + return nil, fmt.Errorf("cannot create user on pds with banned domain") + } + + c := &xrpc.Client{Host: durl.String()} + s.Index.ApplyPDSClientSettings(c) + + if peering.ID == 0 { + // TODO: the case of handling a new user on a new PDS probably requires more thought + cfg, err := atproto.ServerDescribeServer(ctx, c) if err != nil { + // TODO: failing this shouldn't halt our indexing + return nil, fmt.Errorf("failed to check unrecognized pds: %w", err) + } + + // since handles can be anything, checking against this list doesn't matter... + _ = cfg + + // TODO: could check other things, a valid response is good enough for now + peering.Host = durl.Host + peering.SSL = (durl.Scheme == "https") + peering.CrawlRateLimit = float64(s.slurper.DefaultCrawlLimit) + peering.RateLimit = float64(s.slurper.DefaultPerSecondLimit) + peering.HourlyEventLimit = s.slurper.DefaultPerHourLimit + peering.DailyEventLimit = s.slurper.DefaultPerDayLimit + peering.RepoLimit = s.slurper.DefaultRepoLimit + + if s.ssl && !peering.SSL { + return nil, fmt.Errorf("did references non-ssl PDS, this is disallowed in prod: %q %q", did, svc.ServiceEndpoint) + } + + if err := s.db.Create(&peering).Error; err != nil { return nil, err } + } - handle = hurl.Host + if peering.ID == 0 { + panic("somehow failed to create a pds entry?") } - c := &xrpc.Client{Host: durl.String()} - profile, err := bsky.ActorGetProfile(ctx, c, did) + if peering.Blocked { + return nil, fmt.Errorf("refusing to create user with blocked PDS") + } + + if peering.RepoCount >= peering.RepoLimit { + return nil, fmt.Errorf("refusing to create user on PDS at max repo limit for pds %q", peering.Host) + } + + // Increment the repo count for the PDS + res := s.db.Model(&models.PDS{}).Where("id = ? AND repo_count < repo_limit", peering.ID).Update("repo_count", gorm.Expr("repo_count + 1")) + if res.Error != nil { + return nil, fmt.Errorf("failed to increment repo count for pds %q: %w", peering.Host, res.Error) + } + + if res.RowsAffected == 0 { + return nil, fmt.Errorf("refusing to create user on PDS at max repo limit for pds %q", peering.Host) + } + + successfullyCreated := false + + // Release the count if we fail to create the user + defer func() { + if !successfullyCreated { + if err := s.db.Model(&models.PDS{}).Where("id = ?", peering.ID).Update("repo_count", gorm.Expr("repo_count - 1")).Error; err != nil { + s.log.Error("failed to decrement repo count for pds", "err", err) + } + } + }() + + if len(doc.AlsoKnownAs) == 0 { + return nil, fmt.Errorf("user has no 'known as' field in their DID document") + } + + hurl, err := url.Parse(doc.AlsoKnownAs[0]) if err != nil { return nil, err } - if handle != profile.Handle { - return nil, fmt.Errorf("mismatch in handle between did document and pds profile (%s != %s)", handle, profile.Handle) + s.log.Debug("creating external user", "did", did, "handle", hurl.Host, "pds", peering.ID) + + handle := hurl.Host + + validHandle := true + + resdid, err := s.hr.ResolveHandleToDid(ctx, handle) + if err != nil { + s.log.Error("failed to resolve users claimed handle on pds", "handle", handle, "err", err) + validHandle = false + } + + if resdid != did { + s.log.Error("claimed handle did not match servers response", "resdid", resdid, "did", did) + validHandle = false + } + + s.extUserLk.Lock() + defer s.extUserLk.Unlock() + + exu, err := s.Index.LookupUserByDid(ctx, did) + if err == nil { + s.log.Debug("lost the race to create a new user", "did", did, "handle", handle, "existing_hand", exu.Handle) + if exu.PDS != peering.ID { + // User is now on a different PDS, update + if err := s.db.Model(User{}).Where("id = ?", exu.Uid).Update("pds", peering.ID).Error; err != nil { + return nil, fmt.Errorf("failed to update users pds: %w", err) + } + + if err := s.db.Model(models.ActorInfo{}).Where("uid = ?", exu.Uid).Update("pds", peering.ID).Error; err != nil { + return nil, fmt.Errorf("failed to update users pds on actorInfo: %w", err) + } + + exu.PDS = peering.ID + } + + if exu.Handle.String != handle { + // Users handle has changed, update + if err := s.db.Model(User{}).Where("id = ?", exu.Uid).Update("handle", handle).Error; err != nil { + return nil, fmt.Errorf("failed to update users handle: %w", err) + } + + // Update ActorInfos + if err := s.db.Model(models.ActorInfo{}).Where("uid = ?", exu.Uid).Update("handle", handle).Error; err != nil { + return nil, fmt.Errorf("failed to update actorInfos handle: %w", err) + } + + exu.Handle = sql.NullString{String: handle, Valid: true} + } + return exu, nil + } + + if !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, err } // TODO: request this users info from their server to fill out our data... u := User{ - Handle: handle, - Did: did, - PDS: peering.ID, + Did: did, + PDS: peering.ID, + ValidHandle: validHandle, + } + if validHandle { + u.Handle = sql.NullString{String: handle, Valid: true} } if err := s.db.Create(&u).Error; err != nil { - return nil, fmt.Errorf("failed to create other pds user: %w", err) + // If the new user's handle conflicts with an existing user, + // since we just validated the handle for this user, we'll assume + // the existing user no longer has control of the handle + if errors.Is(err, gorm.ErrDuplicatedKey) { + // Get the UID of the existing user + var existingUser User + if err := s.db.Find(&existingUser, "handle = ?", handle).Error; err != nil { + return nil, fmt.Errorf("failed to find existing user: %w", err) + } + + // Set the existing user's handle to NULL and set the valid_handle flag to false + if err := s.db.Model(User{}).Where("id = ?", existingUser.ID).Update("handle", nil).Update("valid_handle", false).Error; err != nil { + return nil, fmt.Errorf("failed to update outdated user's handle: %w", err) + } + + // Do the same thing for the ActorInfo if it exists + if err := s.db.Model(models.ActorInfo{}).Where("uid = ?", existingUser.ID).Update("handle", nil).Update("valid_handle", false).Error; err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + return nil, fmt.Errorf("failed to update outdated actorInfo's handle: %w", err) + } + } + + // Create the new user + if err := s.db.Create(&u).Error; err != nil { + return nil, fmt.Errorf("failed to create user after handle conflict: %w", err) + } + + s.userCache.Remove(did) + } else { + return nil, fmt.Errorf("failed to create other pds user: %w", err) + } } // okay cool, its a user on a server we are peered with // lets make a local record of that user for the future - subj := &types.ActorInfo{ + subj := &models.ActorInfo{ Uid: u.ID, - Handle: handle, - DisplayName: *profile.DisplayName, + DisplayName: "", //*profile.DisplayName, Did: did, - DeclRefCid: profile.Declaration.Cid, Type: "", PDS: peering.ID, + ValidHandle: validHandle, + } + if validHandle { + subj.Handle = sql.NullString{String: handle, Valid: true} } if err := s.db.Create(subj).Error; err != nil { return nil, err } + successfullyCreated = true + return subj, nil } + +func (bgs *BGS) UpdateAccountStatus(ctx context.Context, did string, status string) error { + ctx, span := tracer.Start(ctx, "UpdateAccountStatus") + defer span.End() + + span.SetAttributes( + attribute.String("did", did), + attribute.String("status", status), + ) + + u, err := bgs.lookupUserByDid(ctx, did) + if err != nil { + return err + } + + switch status { + case events.AccountStatusActive: + // Unset the PDS-specific status flags + if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("upstream_status", events.AccountStatusActive).Error; err != nil { + return fmt.Errorf("failed to set user active status: %w", err) + } + u.SetUpstreamStatus(events.AccountStatusActive) + case events.AccountStatusDeactivated: + if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("upstream_status", events.AccountStatusDeactivated).Error; err != nil { + return fmt.Errorf("failed to set user deactivation status: %w", err) + } + u.SetUpstreamStatus(events.AccountStatusDeactivated) + case events.AccountStatusSuspended: + if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("upstream_status", events.AccountStatusSuspended).Error; err != nil { + return fmt.Errorf("failed to set user suspension status: %w", err) + } + u.SetUpstreamStatus(events.AccountStatusSuspended) + case events.AccountStatusTakendown: + if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("upstream_status", events.AccountStatusTakendown).Error; err != nil { + return fmt.Errorf("failed to set user taken down status: %w", err) + } + u.SetUpstreamStatus(events.AccountStatusTakendown) + + if err := bgs.db.Model(&models.ActorInfo{}).Where("uid = ?", u.ID).UpdateColumns(map[string]any{ + "handle": nil, + }).Error; err != nil { + return err + } + case events.AccountStatusDeleted: + if err := bgs.db.Model(&User{}).Where("id = ?", u.ID).UpdateColumns(map[string]any{ + "tombstoned": true, + "handle": nil, + "upstream_status": events.AccountStatusDeleted, + }).Error; err != nil { + return err + } + u.SetUpstreamStatus(events.AccountStatusDeleted) + + if err := bgs.db.Model(&models.ActorInfo{}).Where("uid = ?", u.ID).UpdateColumns(map[string]any{ + "handle": nil, + }).Error; err != nil { + return err + } + + // delete data from carstore + if err := bgs.repoman.TakeDownRepo(ctx, u.ID); err != nil { + // don't let a failure here prevent us from propagating this event + bgs.log.Error("failed to delete user data from carstore", "err", err) + } + } + + return nil +} + +func (bgs *BGS) TakeDownRepo(ctx context.Context, did string) error { + u, err := bgs.lookupUserByDid(ctx, did) + if err != nil { + return err + } + + if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("taken_down", true).Error; err != nil { + return err + } + u.SetTakenDown(true) + + if err := bgs.repoman.TakeDownRepo(ctx, u.ID); err != nil { + return err + } + + if err := bgs.events.TakeDownRepo(ctx, u.ID); err != nil { + return err + } + + return nil +} + +func (bgs *BGS) ReverseTakedown(ctx context.Context, did string) error { + u, err := bgs.lookupUserByDid(ctx, did) + if err != nil { + return err + } + + if err := bgs.db.Model(User{}).Where("id = ?", u.ID).Update("taken_down", false).Error; err != nil { + return err + } + u.SetTakenDown(false) + + return nil +} + +type revCheckResult struct { + ai *models.ActorInfo + err error +} + +func (bgs *BGS) LoadOrStoreResync(pds models.PDS) (PDSResync, bool) { + bgs.pdsResyncsLk.Lock() + defer bgs.pdsResyncsLk.Unlock() + + if r, ok := bgs.pdsResyncs[pds.ID]; ok && r != nil { + return *r, true + } + + r := PDSResync{ + PDS: pds, + Status: "started", + StatusChangedAt: time.Now(), + } + + bgs.pdsResyncs[pds.ID] = &r + + return r, false +} + +func (bgs *BGS) GetResync(pds models.PDS) (PDSResync, bool) { + bgs.pdsResyncsLk.RLock() + defer bgs.pdsResyncsLk.RUnlock() + + if r, ok := bgs.pdsResyncs[pds.ID]; ok { + return *r, true + } + + return PDSResync{}, false +} + +func (bgs *BGS) UpdateResync(resync PDSResync) { + bgs.pdsResyncsLk.Lock() + defer bgs.pdsResyncsLk.Unlock() + + bgs.pdsResyncs[resync.PDS.ID] = &resync +} + +func (bgs *BGS) SetResyncStatus(id uint, status string) PDSResync { + bgs.pdsResyncsLk.Lock() + defer bgs.pdsResyncsLk.Unlock() + + if r, ok := bgs.pdsResyncs[id]; ok { + r.Status = status + r.StatusChangedAt = time.Now() + } + + return *bgs.pdsResyncs[id] +} + +func (bgs *BGS) CompleteResync(resync PDSResync) { + bgs.pdsResyncsLk.Lock() + defer bgs.pdsResyncsLk.Unlock() + + delete(bgs.pdsResyncs, resync.PDS.ID) +} + +func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { + ctx, span := tracer.Start(ctx, "ResyncPDS") + defer span.End() + log := bgs.log.With("pds", pds.Host, "source", "resync_pds") + resync, found := bgs.LoadOrStoreResync(pds) + if found { + return fmt.Errorf("resync already in progress") + } + defer bgs.CompleteResync(resync) + + start := time.Now() + + log.Warn("starting PDS resync") + + host := "http://" + if pds.SSL { + host = "https://" + } + host += pds.Host + + xrpcc := xrpc.Client{Host: host} + bgs.Index.ApplyPDSClientSettings(&xrpcc) + + limiter := rate.NewLimiter(rate.Limit(50), 1) + cursor := "" + limit := int64(500) + + repos := []comatproto.SyncListRepos_Repo{} + + pages := 0 + + resync = bgs.SetResyncStatus(pds.ID, "listing repos") + for { + pages++ + if pages%10 == 0 { + log.Warn("fetching PDS page during resync", "pages", pages, "total_repos", len(repos)) + resync.NumRepoPages = pages + resync.NumRepos = len(repos) + bgs.UpdateResync(resync) + } + if err := limiter.Wait(ctx); err != nil { + log.Error("failed to wait for rate limiter", "error", err) + return fmt.Errorf("failed to wait for rate limiter: %w", err) + } + repoList, err := comatproto.SyncListRepos(ctx, &xrpcc, cursor, limit) + if err != nil { + log.Error("failed to list repos", "error", err) + return fmt.Errorf("failed to list repos: %w", err) + } + + for _, r := range repoList.Repos { + if r != nil { + repos = append(repos, *r) + } + } + + if repoList.Cursor == nil || *repoList.Cursor == "" { + break + } + cursor = *repoList.Cursor + } + + resync.NumRepoPages = pages + resync.NumRepos = len(repos) + bgs.UpdateResync(resync) + + repolistDone := time.Now() + + log.Warn("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) + resync = bgs.SetResyncStatus(pds.ID, "checking revs") + + // run loop over repos with some concurrency + sem := semaphore.NewWeighted(40) + + // Check repo revs against our local copy and enqueue crawls for any that are out of date + for i, r := range repos { + if err := sem.Acquire(ctx, 1); err != nil { + log.Error("failed to acquire semaphore", "error", err) + continue + } + go func(r comatproto.SyncListRepos_Repo) { + defer sem.Release(1) + log := bgs.log.With("did", r.Did, "remote_rev", r.Rev) + // Fetches the user if we have it, otherwise automatically enqueues it for crawling + ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) + if err != nil { + log.Error("failed to get user while resyncing PDS, we can't recrawl it", "error", err) + return + } + + rev, err := bgs.repoman.GetRepoRev(ctx, ai.Uid) + if err != nil { + log.Warn("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) + err := bgs.Index.Crawler.Crawl(ctx, ai) + if err != nil { + log.Error("failed to enqueue crawl for repo during resync", "error", err, "uid", ai.Uid, "did", ai.Did) + } + return + } + + if rev == "" || rev < r.Rev { + log.Warn("recrawling because the repo rev from the PDS is newer than our local repo rev", "local_rev", rev) + err := bgs.Index.Crawler.Crawl(ctx, ai) + if err != nil { + log.Error("failed to enqueue crawl for repo during resync", "error", err, "uid", ai.Uid, "did", ai.Did) + } + return + } + }(r) + if i%100 == 0 { + if i%10_000 == 0 { + log.Warn("checked revs during resync", "num_repos_checked", i, "num_repos_to_crawl", -1, "took", time.Now().Sub(resync.StatusChangedAt)) + } + resync.NumReposChecked = i + bgs.UpdateResync(resync) + } + } + + resync.NumReposChecked = len(repos) + bgs.UpdateResync(resync) + + bgs.log.Warn("enqueued all crawls, exiting resync", "took", time.Now().Sub(start), "num_repos_to_crawl", -1) + + return nil +} diff --git a/bgs/compactor.go b/bgs/compactor.go new file mode 100644 index 000000000..6339d76b0 --- /dev/null +++ b/bgs/compactor.go @@ -0,0 +1,402 @@ +package bgs + +import ( + "context" + "fmt" + "math/rand/v2" + "sync" + "time" + + "github.com/bluesky-social/indigo/carstore" + "github.com/bluesky-social/indigo/models" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" +) + +type queueItem struct { + uid models.Uid + fast bool +} + +// uniQueue is a queue that only allows one instance of a given uid +type uniQueue struct { + q []queueItem + members map[models.Uid]struct{} + lk sync.Mutex +} + +// Append appends a uid to the end of the queue if it doesn't already exist +func (q *uniQueue) Append(uid models.Uid, fast bool) { + q.lk.Lock() + defer q.lk.Unlock() + + if _, ok := q.members[uid]; ok { + return + } + + q.q = append(q.q, queueItem{uid: uid, fast: fast}) + q.members[uid] = struct{}{} + compactionQueueDepth.Inc() +} + +// Prepend prepends a uid to the beginning of the queue if it doesn't already exist +func (q *uniQueue) Prepend(uid models.Uid, fast bool) { + q.lk.Lock() + defer q.lk.Unlock() + + if _, ok := q.members[uid]; ok { + return + } + + q.q = append([]queueItem{{uid: uid, fast: fast}}, q.q...) + q.members[uid] = struct{}{} + compactionQueueDepth.Inc() +} + +// Has returns true if the queue contains the given uid +func (q *uniQueue) Has(uid models.Uid) bool { + q.lk.Lock() + defer q.lk.Unlock() + + _, ok := q.members[uid] + return ok +} + +// Remove removes the given uid from the queue +func (q *uniQueue) Remove(uid models.Uid) { + q.lk.Lock() + defer q.lk.Unlock() + + if _, ok := q.members[uid]; !ok { + return + } + + for i, item := range q.q { + if item.uid == uid { + q.q = append(q.q[:i], q.q[i+1:]...) + break + } + } + + delete(q.members, uid) + compactionQueueDepth.Dec() +} + +// Pop pops the first item off the front of the queue +func (q *uniQueue) Pop() (queueItem, bool) { + q.lk.Lock() + defer q.lk.Unlock() + + if len(q.q) == 0 { + return queueItem{}, false + } + + item := q.q[0] + q.q = q.q[1:] + delete(q.members, item.uid) + + compactionQueueDepth.Dec() + return item, true +} + +// PopRandom pops a random item off the of the queue +// Note: this disrupts the sorted order of the queue and in-order is no longer quite in-order. The randomly popped element is replaced with the last element. +func (q *uniQueue) PopRandom() (queueItem, bool) { + q.lk.Lock() + defer q.lk.Unlock() + + if len(q.q) == 0 { + return queueItem{}, false + } + + var item queueItem + if len(q.q) == 1 { + item = q.q[0] + q.q = nil + } else { + pos := rand.IntN(len(q.q)) + item = q.q[pos] + last := len(q.q) - 1 + q.q[pos] = q.q[last] + q.q = q.q[:last] + } + delete(q.members, item.uid) + + compactionQueueDepth.Dec() + return item, true +} + +type CompactorState struct { + latestUID models.Uid + latestDID string + status string + stats *carstore.CompactionStats +} + +func (cstate *CompactorState) set(uid models.Uid, did, status string, stats *carstore.CompactionStats) { + cstate.latestUID = uid + cstate.latestDID = did + cstate.status = status + cstate.stats = stats +} + +// Compactor is a compactor daemon that compacts repos in the background +type Compactor struct { + q *uniQueue + stateLk sync.RWMutex + exit chan struct{} + requeueInterval time.Duration + requeueLimit int + requeueShardCount int + requeueFast bool + + numWorkers int + wg sync.WaitGroup +} + +type CompactorOptions struct { + RequeueInterval time.Duration + RequeueLimit int + RequeueShardCount int + RequeueFast bool + NumWorkers int +} + +func DefaultCompactorOptions() *CompactorOptions { + return &CompactorOptions{ + RequeueInterval: time.Hour * 4, + RequeueLimit: 0, + RequeueShardCount: 50, + RequeueFast: true, + NumWorkers: 2, + } +} + +func NewCompactor(opts *CompactorOptions) *Compactor { + if opts == nil { + opts = DefaultCompactorOptions() + } + + return &Compactor{ + q: &uniQueue{ + members: make(map[models.Uid]struct{}), + }, + exit: make(chan struct{}), + requeueInterval: opts.RequeueInterval, + requeueLimit: opts.RequeueLimit, + requeueFast: opts.RequeueFast, + requeueShardCount: opts.RequeueShardCount, + numWorkers: opts.NumWorkers, + } +} + +type compactionStats struct { + Completed map[models.Uid]*carstore.CompactionStats + Targets []carstore.CompactionTarget +} + +var errNoReposToCompact = fmt.Errorf("no repos to compact") + +// Start starts the compactor +func (c *Compactor) Start(bgs *BGS) { + log.Info("starting compactor") + c.wg.Add(c.numWorkers) + for i := range c.numWorkers { + strategy := NextInOrder + if i%2 != 0 { + strategy = NextRandom + } + go c.doWork(bgs, strategy) + } + if c.requeueInterval > 0 { + go func() { + log.Info("starting compactor requeue routine", + "interval", c.requeueInterval, + "limit", c.requeueLimit, + "shardCount", c.requeueShardCount, + "fast", c.requeueFast, + ) + + t := time.NewTicker(c.requeueInterval) + for { + select { + case <-c.exit: + return + case <-t.C: + ctx := context.Background() + ctx, span := otel.Tracer("compactor").Start(ctx, "RequeueRoutine") + if err := c.EnqueueAllRepos(ctx, bgs, c.requeueLimit, c.requeueShardCount, c.requeueFast); err != nil { + log.Error("failed to enqueue all repos", "err", err) + } + span.End() + } + } + }() + } +} + +// Shutdown shuts down the compactor +func (c *Compactor) Shutdown() { + log.Info("stopping compactor") + close(c.exit) + c.wg.Wait() + log.Info("compactor stopped") +} + +func (c *Compactor) doWork(bgs *BGS, strategy NextStrategy) { + defer c.wg.Done() + for { + select { + case <-c.exit: + log.Info("compactor worker exiting, no more active compactions running") + return + default: + } + + ctx := context.Background() + start := time.Now() + state, err := c.compactNext(ctx, bgs, strategy) + if err != nil { + if err == errNoReposToCompact { + log.Debug("no repos to compact, waiting and retrying") + time.Sleep(time.Second * 5) + continue + } + log.Error("failed to compact repo", + "err", err, + "uid", state.latestUID, + "repo", state.latestDID, + "status", state.status, + "stats", state.stats, + "duration", time.Since(start), + ) + // Pause for a bit to avoid spamming failed compactions + time.Sleep(time.Millisecond * 100) + } else { + log.Info("compacted repo", + "uid", state.latestUID, + "repo", state.latestDID, + "status", state.status, + "stats", state.stats, + "duration", time.Since(start), + ) + } + } +} + +type NextStrategy int + +const ( + NextInOrder NextStrategy = iota + NextRandom +) + +func (c *Compactor) compactNext(ctx context.Context, bgs *BGS, strategy NextStrategy) (CompactorState, error) { + ctx, span := otel.Tracer("compactor").Start(ctx, "CompactNext") + defer span.End() + + var item queueItem + var ok bool + switch strategy { + case NextRandom: + item, ok = c.q.PopRandom() + default: + item, ok = c.q.Pop() + } + if !ok { + return CompactorState{}, errNoReposToCompact + } + + state := CompactorState{ + latestUID: item.uid, + latestDID: "unknown", + status: "getting_user", + } + + user, err := bgs.lookupUserByUID(ctx, item.uid) + if err != nil { + span.RecordError(err) + state.status = "failed_getting_user" + err := fmt.Errorf("failed to get user %d: %w", item.uid, err) + return state, err + } + + span.SetAttributes(attribute.String("repo", user.Did), attribute.Int("uid", int(item.uid))) + + state.latestDID = user.Did + + start := time.Now() + st, err := bgs.repoman.CarStore().CompactUserShards(ctx, item.uid, item.fast) + if err != nil { + span.RecordError(err) + state.status = "failed_compacting" + err := fmt.Errorf("failed to compact shards for user %d: %w", item.uid, err) + return state, err + } + compactionDuration.Observe(time.Since(start).Seconds()) + + span.SetAttributes( + attribute.Int("shards.deleted", st.ShardsDeleted), + attribute.Int("shards.new", st.NewShards), + attribute.Int("dupes", st.DupeCount), + attribute.Int("shards.skipped", st.SkippedShards), + attribute.Int("refs", st.TotalRefs), + ) + + state.status = "done" + state.stats = st + + return state, nil +} + +func (c *Compactor) EnqueueRepo(ctx context.Context, user *User, fast bool) { + ctx, span := otel.Tracer("compactor").Start(ctx, "EnqueueRepo") + defer span.End() + log.Info("enqueueing compaction for repo", "repo", user.Did, "uid", user.ID, "fast", fast) + c.q.Append(user.ID, fast) +} + +// EnqueueAllRepos enqueues all repos for compaction +// lim is the maximum number of repos to enqueue +// shardCount is the number of shards to compact per user (0 = default of 50) +// fast is whether to use the fast compaction method (skip large shards) +func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shardCount int, fast bool) error { + ctx, span := otel.Tracer("compactor").Start(ctx, "EnqueueAllRepos") + defer span.End() + + span.SetAttributes( + attribute.Int("lim", lim), + attribute.Int("shardCount", shardCount), + attribute.Bool("fast", fast), + ) + + if shardCount == 0 { + shardCount = 20 + } + + span.SetAttributes(attribute.Int("clampedShardCount", shardCount)) + + log := log.With("source", "compactor_enqueue_all_repos", "lim", lim, "shardCount", shardCount, "fast", fast) + log.Info("enqueueing all repos") + + repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, shardCount) + if err != nil { + return fmt.Errorf("failed to get repos to compact: %w", err) + } + + span.SetAttributes(attribute.Int("repos", len(repos))) + + if lim > 0 && len(repos) > lim { + repos = repos[:lim] + } + + span.SetAttributes(attribute.Int("clampedRepos", len(repos))) + + for _, r := range repos { + c.q.Append(r.Usr, fast) + } + + log.Info("done enqueueing all repos", "repos_enqueued", len(repos)) + + return nil +} diff --git a/bgs/fedmgr.go b/bgs/fedmgr.go index fef0c80a3..b7a1717ed 100644 --- a/bgs/fedmgr.go +++ b/bgs/fedmgr.go @@ -2,21 +2,30 @@ package bgs import ( "context" - "encoding/json" "errors" "fmt" + "log/slog" "math/rand" - "net" + "strings" "sync" "time" + "github.com/RussellLuo/slidingwindow" + comatproto "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/schedulers/parallel" + "github.com/bluesky-social/indigo/models" + "go.opentelemetry.io/otel" + "golang.org/x/time/rate" + "github.com/gorilla/websocket" - "github.com/labstack/gommon/log" + pq "github.com/lib/pq" "gorm.io/gorm" ) -type IndexCallback func(context.Context, *PDS, *events.Event) error +var log = slog.Default().With("system", "bgs") + +type IndexCallback func(context.Context, *models.PDS, *events.XRPCStreamEvent) error // TODO: rename me type Slurper struct { @@ -25,18 +34,336 @@ type Slurper struct { db *gorm.DB lk sync.Mutex - active map[string]*PDS + active map[string]*activeSub + + LimitMux sync.RWMutex + Limiters map[uint]*Limiters + DefaultPerSecondLimit int64 + DefaultPerHourLimit int64 + DefaultPerDayLimit int64 + + DefaultCrawlLimit rate.Limit + DefaultRepoLimit int64 + ConcurrencyPerPDS int64 + MaxQueuePerPDS int64 + + NewPDSPerDayLimiter *slidingwindow.Limiter + + newSubsDisabled bool + trustedDomains []string + + shutdownChan chan bool + shutdownResult chan []error + + ssl bool +} + +type Limiters struct { + PerSecond *slidingwindow.Limiter + PerHour *slidingwindow.Limiter + PerDay *slidingwindow.Limiter +} + +type SlurperOptions struct { + SSL bool + DefaultPerSecondLimit int64 + DefaultPerHourLimit int64 + DefaultPerDayLimit int64 + DefaultCrawlLimit rate.Limit + DefaultRepoLimit int64 + ConcurrencyPerPDS int64 + MaxQueuePerPDS int64 +} + +func DefaultSlurperOptions() *SlurperOptions { + return &SlurperOptions{ + SSL: false, + DefaultPerSecondLimit: 50, + DefaultPerHourLimit: 2500, + DefaultPerDayLimit: 20_000, + DefaultCrawlLimit: rate.Limit(5), + DefaultRepoLimit: 100, + ConcurrencyPerPDS: 100, + MaxQueuePerPDS: 1_000, + } +} + +type activeSub struct { + pds *models.PDS + lk sync.RWMutex + ctx context.Context + cancel func() +} + +func NewSlurper(db *gorm.DB, cb IndexCallback, opts *SlurperOptions) (*Slurper, error) { + if opts == nil { + opts = DefaultSlurperOptions() + } + db.AutoMigrate(&SlurpConfig{}) + s := &Slurper{ + cb: cb, + db: db, + active: make(map[string]*activeSub), + Limiters: make(map[uint]*Limiters), + DefaultPerSecondLimit: opts.DefaultPerSecondLimit, + DefaultPerHourLimit: opts.DefaultPerHourLimit, + DefaultPerDayLimit: opts.DefaultPerDayLimit, + DefaultCrawlLimit: opts.DefaultCrawlLimit, + DefaultRepoLimit: opts.DefaultRepoLimit, + ConcurrencyPerPDS: opts.ConcurrencyPerPDS, + MaxQueuePerPDS: opts.MaxQueuePerPDS, + ssl: opts.SSL, + shutdownChan: make(chan bool), + shutdownResult: make(chan []error), + } + if err := s.loadConfig(); err != nil { + return nil, err + } + + // Start a goroutine to flush cursors to the DB every 30s + go func() { + for { + select { + case <-s.shutdownChan: + log.Info("flushing PDS cursors on shutdown") + ctx := context.Background() + ctx, span := otel.Tracer("feedmgr").Start(ctx, "CursorFlusherShutdown") + defer span.End() + var errs []error + if errs = s.flushCursors(ctx); len(errs) > 0 { + for _, err := range errs { + log.Error("failed to flush cursors on shutdown", "err", err) + } + } + log.Info("done flushing PDS cursors on shutdown") + s.shutdownResult <- errs + return + case <-time.After(time.Second * 10): + log.Debug("flushing PDS cursors") + ctx := context.Background() + ctx, span := otel.Tracer("feedmgr").Start(ctx, "CursorFlusher") + defer span.End() + if errs := s.flushCursors(ctx); len(errs) > 0 { + for _, err := range errs { + log.Error("failed to flush cursors", "err", err) + } + } + log.Debug("done flushing PDS cursors") + } + } + }() + + return s, nil +} + +func windowFunc() (slidingwindow.Window, slidingwindow.StopFunc) { + return slidingwindow.NewLocalWindow() +} + +func (s *Slurper) GetLimiters(pdsID uint) *Limiters { + s.LimitMux.RLock() + defer s.LimitMux.RUnlock() + return s.Limiters[pdsID] +} + +func (s *Slurper) GetOrCreateLimiters(pdsID uint, perSecLimit int64, perHourLimit int64, perDayLimit int64) *Limiters { + s.LimitMux.RLock() + defer s.LimitMux.RUnlock() + lim, ok := s.Limiters[pdsID] + if !ok { + perSec, _ := slidingwindow.NewLimiter(time.Second, perSecLimit, windowFunc) + perHour, _ := slidingwindow.NewLimiter(time.Hour, perHourLimit, windowFunc) + perDay, _ := slidingwindow.NewLimiter(time.Hour*24, perDayLimit, windowFunc) + lim = &Limiters{ + PerSecond: perSec, + PerHour: perHour, + PerDay: perDay, + } + s.Limiters[pdsID] = lim + } + + return lim +} + +func (s *Slurper) SetLimits(pdsID uint, perSecLimit int64, perHourLimit int64, perDayLimit int64) { + s.LimitMux.Lock() + defer s.LimitMux.Unlock() + lim, ok := s.Limiters[pdsID] + if !ok { + perSec, _ := slidingwindow.NewLimiter(time.Second, perSecLimit, windowFunc) + perHour, _ := slidingwindow.NewLimiter(time.Hour, perHourLimit, windowFunc) + perDay, _ := slidingwindow.NewLimiter(time.Hour*24, perDayLimit, windowFunc) + lim = &Limiters{ + PerSecond: perSec, + PerHour: perHour, + PerDay: perDay, + } + s.Limiters[pdsID] = lim + } + + lim.PerSecond.SetLimit(perSecLimit) + lim.PerHour.SetLimit(perHourLimit) + lim.PerDay.SetLimit(perDayLimit) +} + +// Shutdown shuts down the slurper +func (s *Slurper) Shutdown() []error { + s.shutdownChan <- true + log.Info("waiting for slurper shutdown") + errs := <-s.shutdownResult + if len(errs) > 0 { + for _, err := range errs { + log.Error("shutdown error", "err", err) + } + } + log.Info("slurper shutdown complete") + return errs +} + +func (s *Slurper) loadConfig() error { + var sc SlurpConfig + if err := s.db.Find(&sc).Error; err != nil { + return err + } + + if sc.ID == 0 { + if err := s.db.Create(&SlurpConfig{}).Error; err != nil { + return err + } + } + + s.newSubsDisabled = sc.NewSubsDisabled + s.trustedDomains = sc.TrustedDomains + + s.NewPDSPerDayLimiter, _ = slidingwindow.NewLimiter(time.Hour*24, sc.NewPDSPerDayLimit, windowFunc) + + return nil +} + +type SlurpConfig struct { + gorm.Model + + NewSubsDisabled bool + TrustedDomains pq.StringArray `gorm:"type:text[]"` + NewPDSPerDayLimit int64 +} + +func (s *Slurper) SetNewSubsDisabled(dis bool) error { + s.lk.Lock() + defer s.lk.Unlock() + + if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("new_subs_disabled", dis).Error; err != nil { + return err + } + + s.newSubsDisabled = dis + return nil +} + +func (s *Slurper) GetNewSubsDisabledState() bool { + s.lk.Lock() + defer s.lk.Unlock() + return s.newSubsDisabled +} + +func (s *Slurper) SetNewPDSPerDayLimit(limit int64) error { + s.lk.Lock() + defer s.lk.Unlock() + + if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("new_pds_per_day_limit", limit).Error; err != nil { + return err + } + + s.NewPDSPerDayLimiter.SetLimit(limit) + return nil +} + +func (s *Slurper) GetNewPDSPerDayLimit() int64 { + s.lk.Lock() + defer s.lk.Unlock() + return s.NewPDSPerDayLimiter.Limit() +} + +func (s *Slurper) AddTrustedDomain(domain string) error { + s.lk.Lock() + defer s.lk.Unlock() + + if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("trusted_domains", gorm.Expr("array_append(trusted_domains, ?)", domain)).Error; err != nil { + return err + } + + s.trustedDomains = append(s.trustedDomains, domain) + return nil +} + +func (s *Slurper) RemoveTrustedDomain(domain string) error { + s.lk.Lock() + defer s.lk.Unlock() + + if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("trusted_domains", gorm.Expr("array_remove(trusted_domains, ?)", domain)).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + return err + } + + for i, d := range s.trustedDomains { + if d == domain { + s.trustedDomains = append(s.trustedDomains[:i], s.trustedDomains[i+1:]...) + break + } + } + + return nil +} + +func (s *Slurper) SetTrustedDomains(domains []string) error { + s.lk.Lock() + defer s.lk.Unlock() + + if err := s.db.Model(SlurpConfig{}).Where("id = 1").Update("trusted_domains", domains).Error; err != nil { + return err + } + + s.trustedDomains = domains + return nil +} + +func (s *Slurper) GetTrustedDomains() []string { + s.lk.Lock() + defer s.lk.Unlock() + return s.trustedDomains } -func NewSlurper(db *gorm.DB, cb IndexCallback) *Slurper { - return &Slurper{ - cb: cb, - db: db, - active: make(map[string]*PDS), +var ErrNewSubsDisabled = fmt.Errorf("new subscriptions temporarily disabled") + +// Checks whether a host is allowed to be subscribed to +// must be called with the slurper lock held +func (s *Slurper) canSlurpHost(host string) bool { + // Check if we're over the limit for new PDSs today + if !s.NewPDSPerDayLimiter.Allow() { + return false } + + // Check if the host is a trusted domain + for _, d := range s.trustedDomains { + // If the domain starts with a *., it's a wildcard + if strings.HasPrefix(d, "*.") { + // Cut off the * so we have .domain.com + if strings.HasSuffix(host, strings.TrimPrefix(d, "*")) { + return true + } + } else { + if host == d { + return true + } + } + } + + return !s.newSubsDisabled } -func (s *Slurper) SubscribeToPds(ctx context.Context, host string) error { +func (s *Slurper) SubscribeToPds(ctx context.Context, host string, reg bool, adminOverride bool, rateOverrides *PDSRates) error { // TODO: for performance, lock on the hostname instead of global s.lk.Lock() defer s.lk.Unlock() @@ -46,15 +373,36 @@ func (s *Slurper) SubscribeToPds(ctx context.Context, host string) error { return nil } - var peering PDS - if err := s.db.First(&peering, "host = ?", host).Error; err != nil { - if !errors.Is(err, gorm.ErrRecordNotFound) { - return err - } + var peering models.PDS + if err := s.db.Find(&peering, "host = ?", host).Error; err != nil { + return err + } + if peering.Blocked { + return fmt.Errorf("cannot subscribe to blocked pds") + } + + if peering.ID == 0 { + if !adminOverride && !s.canSlurpHost(host) { + return ErrNewSubsDisabled + } // New PDS! - npds := PDS{ - Host: host, + npds := models.PDS{ + Host: host, + SSL: s.ssl, + Registered: reg, + RateLimit: float64(s.DefaultPerSecondLimit), + HourlyEventLimit: s.DefaultPerHourLimit, + DailyEventLimit: s.DefaultPerDayLimit, + CrawlRateLimit: float64(s.DefaultCrawlLimit), + RepoLimit: s.DefaultRepoLimit, + } + if rateOverrides != nil { + npds.RateLimit = float64(rateOverrides.PerSecond) + npds.HourlyEventLimit = rateOverrides.PerHour + npds.DailyEventLimit = rateOverrides.PerDay + npds.CrawlRateLimit = float64(rateOverrides.CrawlRate) + npds.RepoLimit = rateOverrides.RepoLimit } if err := s.db.Create(&npds).Error; err != nil { return err @@ -63,14 +411,57 @@ func (s *Slurper) SubscribeToPds(ctx context.Context, host string) error { peering = npds } - s.active[host] = &peering + if !peering.Registered && reg { + peering.Registered = true + if err := s.db.Model(models.PDS{}).Where("id = ?", peering.ID).Update("registered", true).Error; err != nil { + return err + } + } - go s.subscribeWithRedialer(&peering) + ctx, cancel := context.WithCancel(context.Background()) + sub := activeSub{ + pds: &peering, + ctx: ctx, + cancel: cancel, + } + s.active[host] = &sub + + s.GetOrCreateLimiters(peering.ID, int64(peering.RateLimit), peering.HourlyEventLimit, peering.DailyEventLimit) + + go s.subscribeWithRedialer(ctx, &peering, &sub) return nil } -func (s *Slurper) subscribeWithRedialer(host *PDS) { +func (s *Slurper) RestartAll() error { + s.lk.Lock() + defer s.lk.Unlock() + + var all []models.PDS + if err := s.db.Find(&all, "registered = true AND blocked = false").Error; err != nil { + return err + } + + for _, pds := range all { + pds := pds + + ctx, cancel := context.WithCancel(context.Background()) + sub := activeSub{ + pds: &pds, + ctx: ctx, + cancel: cancel, + } + s.active[pds.Host] = &sub + + // Check if we've already got a limiter for this PDS + s.GetOrCreateLimiters(pds.ID, int64(pds.RateLimit), pds.HourlyEventLimit, pds.DailyEventLimit) + go s.subscribeWithRedialer(ctx, &pds, &sub) + } + + return nil +} + +func (s *Slurper) subscribeWithRedialer(ctx context.Context, host *models.PDS, sub *activeSub) { defer func() { s.lk.Lock() defer s.lk.Unlock() @@ -78,27 +469,66 @@ func (s *Slurper) subscribeWithRedialer(host *PDS) { delete(s.active, host.Host) }() - d := websocket.Dialer{} + d := websocket.Dialer{ + HandshakeTimeout: time.Second * 5, + } + + protocol := "ws" + if s.ssl { + protocol = "wss" + } + + // Special case `.host.bsky.network` PDSs to rewind cursor by 200 events to smooth over unclean shutdowns + if strings.HasSuffix(host.Host, ".host.bsky.network") && host.Cursor > 200 { + host.Cursor -= 200 + } + + cursor := host.Cursor + + connectedInbound.Inc() + defer connectedInbound.Dec() + // TODO:? maybe keep a gauge of 'in retry backoff' sources? var backoff int for { + select { + case <-ctx.Done(): + return + default: + } - con, res, err := d.Dial("ws://"+host.Host+"/events", nil) + url := fmt.Sprintf("%s://%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", protocol, host.Host, cursor) + con, res, err := d.DialContext(ctx, url, nil) if err != nil { - fmt.Printf("dialing %q failed: %s", host.Host, err) + log.Warn("dialing failed", "pdsHost", host.Host, "err", err, "backoff", backoff) time.Sleep(sleepForBackoff(backoff)) backoff++ + + if backoff > 15 { + log.Warn("pds does not appear to be online, disabling for now", "pdsHost", host.Host) + if err := s.db.Model(&models.PDS{}).Where("id = ?", host.ID).Update("registered", false).Error; err != nil { + log.Error("failed to unregister failing pds", "err", err) + } + + return + } + continue } - fmt.Println("event subscription response code: ", res.StatusCode) + log.Info("event subscription response", "code", res.StatusCode) - if err := s.handleConnection(host, con); err != nil { + curCursor := cursor + if err := s.handleConnection(ctx, host, con, &cursor, sub); err != nil { if errors.Is(err, ErrTimeoutShutdown) { - log.Infof("shutting down pds subscription to %s, no activity after %s", host.Host, EventsTimeout) + log.Info("shutting down pds subscription after timeout", "host", host.Host, "time", EventsTimeout) return } - log.Warnf("connection to %q failed: %s", host.Host, err) + log.Warn("connection to failed", "host", host.Host, "err", err) + } + + if cursor > curCursor { + backoff = 0 } } } @@ -119,31 +549,186 @@ var ErrTimeoutShutdown = fmt.Errorf("timed out waiting for new events") var EventsTimeout = time.Minute -func (s *Slurper) handleConnection(host *PDS, con *websocket.Conn) error { - for { - if err := con.SetReadDeadline(time.Now().Add(EventsTimeout)); err != nil { - return fmt.Errorf("failed to set read deadline: %w", err) - } +func (s *Slurper) handleConnection(ctx context.Context, host *models.PDS, con *websocket.Conn, lastCursor *int64, sub *activeSub) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + log.Debug("got remote repo event", "pdsHost", host.Host, "repo", evt.Repo, "seq", evt.Seq) + if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{ + RepoCommit: evt, + }); err != nil { + log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err) + } + *lastCursor = evt.Seq - mt, data, err := con.ReadMessage() - if err != nil { - if nerr, ok := err.(net.Error); ok && nerr.Timeout() { - return ErrTimeoutShutdown + if err := s.updateCursor(sub, *lastCursor); err != nil { + return fmt.Errorf("updating cursor: %w", err) } - return err - } + return nil + }, + RepoSync: func(evt *comatproto.SyncSubscribeRepos_Sync) error { + log.Info("sync event", "did", evt.Did, "pdsHost", host.Host, "seq", evt.Seq) + if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{ + RepoSync: evt, + }); err != nil { + log.Error("failed handling event", "host", host.Host, "seq", evt.Seq, "err", err) + } + *lastCursor = evt.Seq + + if err := s.updateCursor(sub, *lastCursor); err != nil { + return fmt.Errorf("updating cursor: %w", err) + } + + return nil + }, + RepoInfo: func(info *comatproto.SyncSubscribeRepos_Info) error { + log.Info("info event", "name", info.Name, "message", info.Message, "pdsHost", host.Host) + return nil + }, + RepoIdentity: func(ident *comatproto.SyncSubscribeRepos_Identity) error { + log.Info("identity event", "did", ident.Did) + if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{ + RepoIdentity: ident, + }); err != nil { + log.Error("failed handling event", "host", host.Host, "seq", ident.Seq, "err", err) + } + *lastCursor = ident.Seq + + if err := s.updateCursor(sub, *lastCursor); err != nil { + return fmt.Errorf("updating cursor: %w", err) + } + + return nil + }, + RepoAccount: func(acct *comatproto.SyncSubscribeRepos_Account) error { + log.Info("account event", "did", acct.Did, "status", acct.Status) + if err := s.cb(context.TODO(), host, &events.XRPCStreamEvent{ + RepoAccount: acct, + }); err != nil { + log.Error("failed handling event", "host", host.Host, "seq", acct.Seq, "err", err) + } + *lastCursor = acct.Seq + + if err := s.updateCursor(sub, *lastCursor); err != nil { + return fmt.Errorf("updating cursor: %w", err) + } + + return nil + }, + // TODO: all the other event types (handle change, migration, etc) + Error: func(errf *events.ErrorFrame) error { + switch errf.Error { + case "FutureCursor": + // if we get a FutureCursor frame, reset our sequence number for this host + if err := s.db.Table("pds").Where("id = ?", host.ID).Update("cursor", 0).Error; err != nil { + return err + } + + *lastCursor = 0 + return fmt.Errorf("got FutureCursor frame, reset cursor tracking for host") + default: + return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message) + } + }, + } + + lims := s.GetOrCreateLimiters(host.ID, int64(host.RateLimit), host.HourlyEventLimit, host.DailyEventLimit) + + limiters := []*slidingwindow.Limiter{ + lims.PerSecond, + lims.PerHour, + lims.PerDay, + } + + instrumentedRSC := events.NewInstrumentedRepoStreamCallbacks(limiters, rsc.EventHandler) + + pool := parallel.NewScheduler( + 100, + 1_000, + con.RemoteAddr().String(), + instrumentedRSC.EventHandler, + ) + return events.HandleRepoStream(ctx, con, pool, nil) +} + +func (s *Slurper) updateCursor(sub *activeSub, curs int64) error { + sub.lk.Lock() + defer sub.lk.Unlock() + sub.pds.Cursor = curs + return nil +} + +type cursorSnapshot struct { + id uint + cursor int64 +} + +// flushCursors updates the PDS cursors in the DB for all active subscriptions +func (s *Slurper) flushCursors(ctx context.Context) []error { + ctx, span := otel.Tracer("feedmgr").Start(ctx, "flushCursors") + defer span.End() + + var cursors []cursorSnapshot + + s.lk.Lock() + // Iterate over active subs and copy the current cursor + for _, sub := range s.active { + sub.lk.RLock() + cursors = append(cursors, cursorSnapshot{ + id: sub.pds.ID, + cursor: sub.pds.Cursor, + }) + sub.lk.RUnlock() + } + s.lk.Unlock() - _ = mt + errs := []error{} - var ev events.Event - if err := json.Unmarshal(data, &ev); err != nil { - return fmt.Errorf("failed to unmarshal event: %w", err) + tx := s.db.WithContext(ctx).Begin() + for _, cursor := range cursors { + if err := tx.WithContext(ctx).Model(models.PDS{}).Where("id = ?", cursor.id).UpdateColumn("cursor", cursor.cursor).Error; err != nil { + errs = append(errs, err) } + } + if err := tx.WithContext(ctx).Commit().Error; err != nil { + errs = append(errs, err) + } - fmt.Println("got event: ", host.Host, ev.Kind) - if err := s.cb(context.TODO(), host, &ev); err != nil { - log.Errorf("failed to index event from %q: %s", host.Host, err) + return errs +} + +func (s *Slurper) GetActiveList() []string { + s.lk.Lock() + defer s.lk.Unlock() + var out []string + for k := range s.active { + out = append(out, k) + } + + return out +} + +var ErrNoActiveConnection = fmt.Errorf("no active connection to host") + +func (s *Slurper) KillUpstreamConnection(host string, block bool) error { + s.lk.Lock() + defer s.lk.Unlock() + + ac, ok := s.active[host] + if !ok { + return fmt.Errorf("killing connection %q: %w", host, ErrNoActiveConnection) + } + ac.cancel() + // cleanup in the run thread subscribeWithRedialer() will delete(s.active, host) + + if block { + if err := s.db.Model(models.PDS{}).Where("id = ?", ac.pds.ID).UpdateColumn("blocked", true).Error; err != nil { + return fmt.Errorf("failed to set host as blocked: %w", err) } } + + return nil } diff --git a/bgs/handlers.go b/bgs/handlers.go new file mode 100644 index 000000000..ef5284bd7 --- /dev/null +++ b/bgs/handlers.go @@ -0,0 +1,320 @@ +package bgs + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + atproto "github.com/bluesky-social/indigo/api/atproto" + comatprototypes "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/carstore" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/mst" + "gorm.io/gorm" + + "github.com/bluesky-social/indigo/xrpc" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipld/go-car" + "github.com/labstack/echo/v4" +) + +func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection string, did string, rkey string) (io.Reader, error) { + u, err := s.lookupUserByDid(ctx, did) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, echo.NewHTTPError(http.StatusNotFound, "user not found") + } + log.Error("failed to lookup user", "err", err, "did", did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user") + } + + if u.GetTombstoned() { + return nil, fmt.Errorf("account was deleted") + } + + if u.GetTakenDown() { + return nil, fmt.Errorf("account was taken down by the Relay") + } + + ustatus := u.GetUpstreamStatus() + if ustatus == events.AccountStatusTakendown { + return nil, fmt.Errorf("account was taken down by its PDS") + } + + if ustatus == events.AccountStatusDeactivated { + return nil, fmt.Errorf("account is temporarily deactivated") + } + + if ustatus == events.AccountStatusSuspended { + return nil, fmt.Errorf("account is suspended by its PDS") + } + + root, blocks, err := s.repoman.GetRecordProof(ctx, u.ID, collection, rkey) + if err != nil { + if errors.Is(err, mst.ErrNotFound) { + return nil, echo.NewHTTPError(http.StatusNotFound, "record not found in repo") + } + log.Error("failed to get record from repo", "err", err, "did", did, "collection", collection, "rkey", rkey) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get record from repo") + } + + buf := new(bytes.Buffer) + hb, err := cbor.DumpObject(&car.CarHeader{ + Roots: []cid.Cid{root}, + Version: 1, + }) + if _, err := carstore.LdWrite(buf, hb); err != nil { + return nil, err + } + + for _, blk := range blocks { + if _, err := carstore.LdWrite(buf, blk.Cid().Bytes(), blk.RawData()); err != nil { + return nil, err + } + } + + return buf, nil +} + +func (s *BGS) handleComAtprotoSyncGetRepo(ctx context.Context, did string, since string) (io.Reader, error) { + u, err := s.lookupUserByDid(ctx, did) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, echo.NewHTTPError(http.StatusNotFound, "user not found") + } + log.Error("failed to lookup user", "err", err, "did", did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user") + } + + if u.GetTombstoned() { + return nil, fmt.Errorf("account was deleted") + } + + if u.GetTakenDown() { + return nil, fmt.Errorf("account was taken down by the Relay") + } + + ustatus := u.GetUpstreamStatus() + if ustatus == events.AccountStatusTakendown { + return nil, fmt.Errorf("account was taken down by its PDS") + } + + if ustatus == events.AccountStatusDeactivated { + return nil, fmt.Errorf("account is temporarily deactivated") + } + + if ustatus == events.AccountStatusSuspended { + return nil, fmt.Errorf("account is suspended by its PDS") + } + + // TODO: stream the response + buf := new(bytes.Buffer) + if err := s.repoman.ReadRepo(ctx, u.ID, since, buf); err != nil { + log.Error("failed to read repo into buffer", "err", err, "did", did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to read repo into buffer") + } + + return buf, nil +} + +func (s *BGS) handleComAtprotoSyncGetBlocks(ctx context.Context, cids []string, did string) (io.Reader, error) { + return nil, fmt.Errorf("NYI") +} + +func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context, body *comatprototypes.SyncRequestCrawl_Input) error { + host := body.Hostname + if host == "" { + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname") + } + + if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { + if s.ssl { + host = "https://" + host + } else { + host = "http://" + host + } + } + + u, err := url.Parse(host) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "failed to parse hostname") + } + + if u.Scheme == "http" && s.ssl { + return echo.NewHTTPError(http.StatusBadRequest, "this server requires https") + } + + if u.Scheme == "https" && !s.ssl { + return echo.NewHTTPError(http.StatusBadRequest, "this server does not support https") + } + + if u.Path != "" { + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without path") + } + + if u.Query().Encode() != "" { + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname without query") + } + + host = u.Host // potentially hostname:port + + banned, err := s.domainIsBanned(ctx, host) + if banned { + return echo.NewHTTPError(http.StatusUnauthorized, "domain is banned") + } + + log.Warn("TODO: better host validation for crawl requests") + + clientHost := fmt.Sprintf("%s://%s", u.Scheme, host) + + c := &xrpc.Client{ + Host: clientHost, + Client: http.DefaultClient, // not using the client that auto-retries + } + + desc, err := atproto.ServerDescribeServer(ctx, c) + if err != nil { + errMsg := fmt.Sprintf("requested host (%s) failed to respond to describe request", clientHost) + return echo.NewHTTPError(http.StatusBadRequest, errMsg) + } + + // Maybe we could do something with this response later + _ = desc + + if len(s.nextCrawlers) != 0 { + blob, err := json.Marshal(body) + if err != nil { + log.Warn("could not forward requestCrawl, json err", "err", err) + } else { + go func(bodyBlob []byte) { + for _, rpu := range s.nextCrawlers { + pu := rpu.JoinPath("/xrpc/com.atproto.sync.requestCrawl") + response, err := s.httpClient.Post(pu.String(), "application/json", bytes.NewReader(bodyBlob)) + if response != nil && response.Body != nil { + response.Body.Close() + } + if err != nil || response == nil { + log.Warn("requestCrawl forward failed", "host", rpu, "err", err) + } else if response.StatusCode != http.StatusOK { + log.Warn("requestCrawl forward failed", "host", rpu, "status", response.Status) + } else { + log.Info("requestCrawl forward successful", "host", rpu) + } + } + }(blob) + } + } + + return s.slurper.SubscribeToPds(ctx, host, true, false, nil) +} + +func (s *BGS) handleComAtprotoSyncNotifyOfUpdate(ctx context.Context, body *comatprototypes.SyncNotifyOfUpdate_Input) error { + // TODO: + return nil +} + +func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor int64, limit int) (*comatprototypes.SyncListRepos_Output, error) { + // Filter out tombstoned, taken down, and deactivated accounts + q := fmt.Sprintf("id > ? AND NOT tombstoned AND NOT taken_down AND (upstream_status is NULL OR (upstream_status != '%s' AND upstream_status != '%s' AND upstream_status != '%s'))", + events.AccountStatusDeactivated, events.AccountStatusSuspended, events.AccountStatusTakendown) + + // Load the users + users := []*User{} + if err := s.db.Model(&User{}).Where(q, cursor).Order("id").Limit(limit).Find(&users).Error; err != nil { + if err == gorm.ErrRecordNotFound { + return &comatprototypes.SyncListRepos_Output{}, nil + } + log.Error("failed to query users", "err", err) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to query users") + } + + if len(users) == 0 { + // resp.Repos is an explicit empty array, not just 'nil' + return &comatprototypes.SyncListRepos_Output{ + Repos: []*comatprototypes.SyncListRepos_Repo{}, + }, nil + } + + resp := &comatprototypes.SyncListRepos_Output{ + Repos: make([]*comatprototypes.SyncListRepos_Repo, len(users)), + } + + // Fetch the repo roots for each user + for i := range users { + user := users[i] + + root, err := s.repoman.GetRepoRoot(ctx, user.ID) + if err != nil { + log.Error("failed to get repo root", "err", err, "did", user.Did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("failed to get repo root for (%s): %v", user.Did, err.Error())) + } + + resp.Repos[i] = &comatprototypes.SyncListRepos_Repo{ + Did: user.Did, + Head: root.String(), + } + } + + // If this is not the last page, set the cursor + if len(users) >= limit && len(users) > 1 { + nextCursor := fmt.Sprintf("%d", users[len(users)-1].ID) + resp.Cursor = &nextCursor + } + + return resp, nil +} + +func (s *BGS) handleComAtprotoSyncGetLatestCommit(ctx context.Context, did string) (*comatprototypes.SyncGetLatestCommit_Output, error) { + u, err := s.lookupUserByDid(ctx, did) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, echo.NewHTTPError(http.StatusNotFound, "user not found") + } + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user") + } + + if u.GetTombstoned() { + return nil, fmt.Errorf("account was deleted") + } + + if u.GetTakenDown() { + return nil, fmt.Errorf("account was taken down by the Relay") + } + + ustatus := u.GetUpstreamStatus() + if ustatus == events.AccountStatusTakendown { + return nil, fmt.Errorf("account was taken down by its PDS") + } + + if ustatus == events.AccountStatusDeactivated { + return nil, fmt.Errorf("account is temporarily deactivated") + } + + if ustatus == events.AccountStatusSuspended { + return nil, fmt.Errorf("account is suspended by its PDS") + } + + root, err := s.repoman.GetRepoRoot(ctx, u.ID) + if err != nil { + log.Error("failed to get repo root", "err", err, "did", u.Did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get repo root") + } + + rev, err := s.repoman.GetRepoRev(ctx, u.ID) + if err != nil { + log.Error("failed to get repo rev", "err", err, "did", u.Did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get repo rev") + } + + return &comatprototypes.SyncGetLatestCommit_Output{ + Cid: root.String(), + Rev: rev, + }, nil +} diff --git a/bgs/metrics.go b/bgs/metrics.go new file mode 100644 index 000000000..519cd2e30 --- /dev/null +++ b/bgs/metrics.go @@ -0,0 +1,75 @@ +package bgs + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var eventsReceivedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "events_received_counter", + Help: "The total number of events received", +}, []string{"pds"}) + +var eventsHandleDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "events_handle_duration", + Help: "A histogram of handleFedEvent latencies", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), +}, []string{"pds"}) + +var repoCommitsReceivedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "repo_commits_received_counter", + Help: "The total number of events received", +}, []string{"pds"}) + +var repoCommitsResultCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "repo_commits_result_counter", + Help: "The results of commit events received", +}, []string{"pds", "status"}) + +var rebasesCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "event_rebases", + Help: "The total number of rebase events received", +}, []string{"pds"}) + +var eventsSentCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "events_sent_counter", + Help: "The total number of events sent to consumers", +}, []string{"remote_addr", "user_agent"}) + +var externalUserCreationAttempts = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_external_user_creation_attempts", + Help: "The total number of external users created", +}) + +var connectedInbound = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "bgs_connected_inbound", + Help: "Number of inbound firehoses we are consuming", +}) + +var compactionDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "compaction_duration", + Help: "A histogram of compaction latencies", + Buckets: prometheus.ExponentialBuckets(0.001, 3, 14), +}) + +var compactionQueueDepth = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "compaction_queue_depth", + Help: "The current depth of the compaction queue", +}) + +var newUsersDiscovered = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_new_users_discovered", + Help: "The total number of new users discovered directly from the firehose (not from refs)", +}) + +var userLookupDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "relay_user_lookup_duration", + Help: "A histogram of user lookup latencies", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), +}) + +var newUserDiscoveryDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "relay_new_user_discovery_duration", + Help: "A histogram of new user discovery latencies", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), +}) diff --git a/bgs/stubs.go b/bgs/stubs.go new file mode 100644 index 000000000..77e6ab509 --- /dev/null +++ b/bgs/stubs.go @@ -0,0 +1,185 @@ +package bgs + +import ( + "fmt" + "io" + "net/http" + "strconv" + + comatprototypes "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/ipfs/go-cid" + "github.com/labstack/echo/v4" + "go.opentelemetry.io/otel" +) + +type XRPCError struct { + Message string `json:"message"` +} + +func (s *BGS) HandleComAtprotoSyncGetBlocks(c echo.Context) error { + ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncGetBlocks") + defer span.End() + + cids := c.QueryParams()["cids"] + did := c.QueryParam("did") + _, err := syntax.ParseDID(did) + if err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid did: %s", did)}) + } + + for _, cd := range cids { + _, err = cid.Parse(cd) + if err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid cid: %s", cd)}) + } + } + + var out io.Reader + var handleErr error + // func (s *BGS) handleComAtprotoSyncGetBlocks(ctx context.Context,cids []string,did string) (io.Reader, error) + out, handleErr = s.handleComAtprotoSyncGetBlocks(ctx, cids, did) + if handleErr != nil { + return handleErr + } + return c.Stream(200, "application/vnd.ipld.car", out) +} + +func (s *BGS) HandleComAtprotoSyncGetLatestCommit(c echo.Context) error { + ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncGetLatestCommit") + defer span.End() + did := c.QueryParam("did") + + _, err := syntax.ParseDID(did) + if err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid did: %s", did)}) + } + + var out *comatprototypes.SyncGetLatestCommit_Output + var handleErr error + // func (s *BGS) handleComAtprotoSyncGetLatestCommit(ctx context.Context,did string) (*comatprototypes.SyncGetLatestCommit_Output, error) + out, handleErr = s.handleComAtprotoSyncGetLatestCommit(ctx, did) + if handleErr != nil { + return handleErr + } + return c.JSON(200, out) +} + +func (s *BGS) HandleComAtprotoSyncGetRecord(c echo.Context) error { + ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncGetRecord") + defer span.End() + collection := c.QueryParam("collection") + did := c.QueryParam("did") + rkey := c.QueryParam("rkey") + + _, err := syntax.ParseRecordKey(rkey) + if err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid rkey: %s", rkey)}) + } + + _, err = syntax.ParseNSID(collection) + if err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid collection: %s", collection)}) + } + + _, err = syntax.ParseDID(did) + if err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid did: %s", did)}) + } + + var out io.Reader + var handleErr error + // func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context,collection string,commit string,did string,rkey string) (io.Reader, error) + out, handleErr = s.handleComAtprotoSyncGetRecord(ctx, collection, did, rkey) + if handleErr != nil { + return handleErr + } + return c.Stream(200, "application/vnd.ipld.car", out) +} + +func (s *BGS) HandleComAtprotoSyncGetRepo(c echo.Context) error { + ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncGetRepo") + defer span.End() + did := c.QueryParam("did") + since := c.QueryParam("since") + + _, err := syntax.ParseDID(did) + if err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid did: %s", did)}) + } + + var out io.Reader + var handleErr error + // func (s *BGS) handleComAtprotoSyncGetRepo(ctx context.Context,did string,since string) (io.Reader, error) + out, handleErr = s.handleComAtprotoSyncGetRepo(ctx, did, since) + if handleErr != nil { + return handleErr + } + return c.Stream(200, "application/vnd.ipld.car", out) +} + +func (s *BGS) HandleComAtprotoSyncListRepos(c echo.Context) error { + ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncListRepos") + defer span.End() + + cursorQuery := c.QueryParam("cursor") + limitQuery := c.QueryParam("limit") + + var err error + + limit := 500 + if limitQuery != "" { + limit, err = strconv.Atoi(limitQuery) + if err != nil || limit < 1 || limit > 1000 { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid limit: %s", limitQuery)}) + } + } + + cursor := int64(0) + if cursorQuery != "" { + cursor, err = strconv.ParseInt(cursorQuery, 10, 64) + if err != nil || cursor < 0 { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid cursor: %s", cursorQuery)}) + } + } + + out, handleErr := s.handleComAtprotoSyncListRepos(ctx, cursor, limit) + if handleErr != nil { + return handleErr + } + return c.JSON(200, out) +} + +func (s *BGS) HandleComAtprotoSyncNotifyOfUpdate(c echo.Context) error { + ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncNotifyOfUpdate") + defer span.End() + + var body comatprototypes.SyncNotifyOfUpdate_Input + if err := c.Bind(&body); err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid body: %s", err)}) + } + var handleErr error + // func (s *BGS) handleComAtprotoSyncNotifyOfUpdate(ctx context.Context,body *comatprototypes.SyncNotifyOfUpdate_Input) error + handleErr = s.handleComAtprotoSyncNotifyOfUpdate(ctx, &body) + if handleErr != nil { + return handleErr + } + return nil +} + +func (s *BGS) HandleComAtprotoSyncRequestCrawl(c echo.Context) error { + ctx, span := otel.Tracer("server").Start(c.Request().Context(), "HandleComAtprotoSyncRequestCrawl") + defer span.End() + + var body comatprototypes.SyncRequestCrawl_Input + if err := c.Bind(&body); err != nil { + return c.JSON(http.StatusBadRequest, XRPCError{Message: fmt.Sprintf("invalid body: %s", err)}) + } + var handleErr error + // func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context,body *comatprototypes.SyncRequestCrawl_Input) error + handleErr = s.handleComAtprotoSyncRequestCrawl(ctx, &body) + if handleErr != nil { + return handleErr + } + return nil +} diff --git a/carstore/README.md b/carstore/README.md new file mode 100644 index 000000000..90880defb --- /dev/null +++ b/carstore/README.md @@ -0,0 +1,41 @@ +# Carstore + +Store a zillion users of PDS-like repo, with more limited operations (mainly: firehose in, firehose out). + +## [ScyllaStore](scylla.go) + +Blocks stored in ScyllaDB. +User and PDS metadata stored in gorm (PostgreSQL or sqlite3). + +## [FileCarStore](bs.go) + +Store 'car slices' from PDS source subscribeRepo firehose streams to filesystem. +Store metadata to gorm postgresql (or sqlite3). +Periodic compaction of car slices into fewer larger car slices. +User and PDS metadata stored in gorm (PostgreSQL or sqlite3). +FileCarStore was the first production carstore and used through at least 2024-11. + +## [SQLiteStore](sqlite_store.go) + +Experimental/demo. +Blocks stored in trivial local sqlite3 schema. +Minimal reference implementation from which fancy scalable/performant implementations may be derived. + +```sql +CREATE TABLE IF NOT EXISTS blocks (uid int, cid blob, rev varchar, root blob, block blob, PRIMARY KEY(uid,cid)) +CREATE INDEX IF NOT EXISTS blocx_by_rev ON blocks (uid, rev DESC) + +INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?) ON CONFLICT (uid,cid) DO UPDATE SET rev=excluded.rev, root=excluded.root, block=excluded.block + +SELECT rev, root FROM blocks WHERE uid = ? ORDER BY rev DESC LIMIT 1 + +SELECT cid,rev,root,block FROM blocks WHERE uid = ? AND rev > ? ORDER BY rev DESC + +DELETE FROM blocks WHERE uid = ? + +SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1 + +SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1 + +SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1 +``` diff --git a/carstore/bs.go b/carstore/bs.go index 8d4d64a04..aacbc1c80 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -4,81 +4,116 @@ import ( "bufio" "bytes" "context" - "encoding/binary" "fmt" "io" + "log/slog" "os" "path/filepath" - "sync" + "sort" + "sync/atomic" + "time" - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-car/util" + "github.com/bluesky-social/indigo/models" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + + blockformat "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" blockstore "github.com/ipfs/go-ipfs-blockstore" cbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" car "github.com/ipld/go-car" + carutil "github.com/ipld/go-car/util" + cbg "github.com/whyrusleeping/cbor-gen" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "gorm.io/gorm" ) -type CarStore struct { - meta *gorm.DB - rootDir string +var blockGetTotalCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "carstore_block_get_total", + Help: "carstore get queries", +}, []string{"usrskip", "cache"}) + +var blockGetTotalCounterUsrskip = blockGetTotalCounter.WithLabelValues("true", "miss") +var blockGetTotalCounterCached = blockGetTotalCounter.WithLabelValues("false", "hit") +var blockGetTotalCounterNormal = blockGetTotalCounter.WithLabelValues("false", "miss") + +const MaxSliceLength = 2 << 20 + +const BigShardThreshold = 2 << 20 + +type CarStore interface { + // TODO: not really part of general interface + CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) + // TODO: not really part of general interface + GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) + + GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) + GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) + ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) + NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) + ReadOnlySession(user models.Uid) (*DeltaSession, error) + ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error + Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) + WipeUserData(ctx context.Context, user models.Uid) error +} + +type FileCarStore struct { + meta *CarStoreGormMeta + rootDirs []string + + lastShardCache lastShardCache - lscLk sync.Mutex - lastShardCache map[uint]*CarShard + log *slog.Logger } -func NewCarStore(meta *gorm.DB, root string) (*CarStore, error) { - if _, err := os.Stat(root); err != nil { - if !os.IsNotExist(err) { - return nil, err - } +func NewCarStore(meta *gorm.DB, roots []string) (CarStore, error) { + for _, root := range roots { + if _, err := os.Stat(root); err != nil { + if !os.IsNotExist(err) { + return nil, err + } - if err := os.Mkdir(root, 0775); err != nil { - return nil, err + if err := os.Mkdir(root, 0775); err != nil { + return nil, err + } } } if err := meta.AutoMigrate(&CarShard{}, &blockRef{}); err != nil { return nil, err } - return &CarStore{ - meta: meta, - rootDir: root, - lastShardCache: make(map[uint]*CarShard), - }, nil -} - -type UserInfo struct { - gorm.Model - Head string -} - -type CarShard struct { - gorm.Model + if err := meta.AutoMigrate(&staleRef{}); err != nil { + return nil, err + } - Root string - DataStart int64 - Seq int `gorm:"index"` - Path string - Usr uint `gorm:"index"` + gormMeta := &CarStoreGormMeta{meta: meta} + out := &FileCarStore{ + meta: gormMeta, + rootDirs: roots, + lastShardCache: lastShardCache{ + source: gormMeta, + }, + log: slog.Default().With("system", "carstore"), + } + out.lastShardCache.Init() + return out, nil } -type blockRef struct { - ID uint `gorm:"primarykey"` - Cid string `gorm:"index"` - Shard uint - Offset int64 - //User uint `gorm:"index"` +// userView needs these things to get into the underlying block store +// implemented by CarStoreGormMeta +type userViewSource interface { + HasUidCid(ctx context.Context, user models.Uid, k cid.Cid) (bool, error) + LookupBlockRef(ctx context.Context, k cid.Cid) (path string, offset int64, user models.Uid, err error) } +// wrapper into a block store that keeps track of which user we are working on behalf of type userView struct { - cs *CarStore - user uint + cs userViewSource + user models.Uid - cache map[cid.Cid]blocks.Block + cache map[cid.Cid]blockformat.Block prefetch bool } @@ -89,59 +124,78 @@ func (uv *userView) HashOnRead(hor bool) { } func (uv *userView) Has(ctx context.Context, k cid.Cid) (bool, error) { - var count int64 - if err := uv.cs.meta. - Model(blockRef{}). - Select("path, block_refs.offset"). - Joins("left join car_shards on block_refs.shard = car_shards.id"). - Where("usr = ? AND cid = ?", uv.user, k.String()). - Count(&count).Error; err != nil { - return false, err + _, have := uv.cache[k] + if have { + return have, nil } - - return count > 0, nil + return uv.cs.HasUidCid(ctx, uv.user, k) } -func (uv *userView) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { +var CacheHits int64 +var CacheMiss int64 + +func (uv *userView) Get(ctx context.Context, k cid.Cid) (blockformat.Block, error) { + + if !k.Defined() { + return nil, fmt.Errorf("attempted to 'get' undefined cid") + } if uv.cache != nil { blk, ok := uv.cache[k] if ok { + blockGetTotalCounterCached.Add(1) + atomic.AddInt64(&CacheHits, 1) + return blk, nil } } + atomic.AddInt64(&CacheMiss, 1) - // TODO: for now, im using a join to ensure we only query blocks from the - // correct user. maybe it makes sense to put the user in the blockRef - // directly? tradeoff of time vs space - var info struct { - Path string - Offset int64 + path, offset, user, err := uv.cs.LookupBlockRef(ctx, k) + if err != nil { + return nil, err + } + if path == "" { + return nil, ipld.ErrNotFound{Cid: k} } - if err := uv.cs.meta. - Model(blockRef{}). - Select("path, block_refs.offset"). - Joins("left join car_shards on block_refs.shard = car_shards.id"). - Where("usr = ? AND cid = ?", uv.user, k.String()). - First(&info).Error; err != nil { - if err == gorm.ErrRecordNotFound { - return nil, ipld.ErrNotFound{k} - } - return nil, err + prefetch := uv.prefetch + if user != uv.user { + blockGetTotalCounterUsrskip.Add(1) + prefetch = false + } else { + blockGetTotalCounterNormal.Add(1) } - if uv.prefetch { - return uv.prefetchRead(ctx, k, info.Path, info.Offset) + if prefetch { + return uv.prefetchRead(ctx, k, path, offset) } else { - return uv.singleRead(ctx, k, info.Path, info.Offset) + return uv.singleRead(ctx, k, path, offset) } } -func (uv *userView) prefetchRead(ctx context.Context, k cid.Cid, path string, offset int64) (blocks.Block, error) { +const prefetchThreshold = 512 << 10 + +func (uv *userView) prefetchRead(ctx context.Context, k cid.Cid, path string, offset int64) (blockformat.Block, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "getLastShard") + defer span.End() + fi, err := os.Open(path) if err != nil { return nil, err } + defer fi.Close() + + st, err := fi.Stat() + if err != nil { + return nil, fmt.Errorf("stat file for prefetch: %w", err) + } + + span.SetAttributes(attribute.Int64("shard_size", st.Size())) + + if st.Size() > prefetchThreshold { + span.SetAttributes(attribute.Bool("no_prefetch", true)) + return doBlockRead(fi, k, offset) + } cr, err := car.NewCarReader(fi) if err != nil { @@ -168,12 +222,17 @@ func (uv *userView) prefetchRead(ctx context.Context, k cid.Cid, path string, of return outblk, nil } -func (uv *userView) singleRead(ctx context.Context, k cid.Cid, path string, offset int64) (blocks.Block, error) { +func (uv *userView) singleRead(ctx context.Context, k cid.Cid, path string, offset int64) (blockformat.Block, error) { fi, err := os.Open(path) if err != nil { return nil, err } + defer fi.Close() + + return doBlockRead(fi, k, offset) +} +func doBlockRead(fi *os.File, k cid.Cid, offset int64) (blockformat.Block, error) { seeked, err := fi.Seek(offset, io.SeekStart) if err != nil { return nil, err @@ -184,7 +243,7 @@ func (uv *userView) singleRead(ctx context.Context, k cid.Cid, path string, offs } bufr := bufio.NewReader(fi) - rcid, data, err := util.ReadNode(bufr) + rcid, data, err := carutil.ReadNode(bufr) if err != nil { return nil, err } @@ -200,11 +259,11 @@ func (uv *userView) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return nil, fmt.Errorf("not implemented") } -func (uv *userView) Put(ctx context.Context, blk blocks.Block) error { +func (uv *userView) Put(ctx context.Context, blk blockformat.Block) error { return fmt.Errorf("puts not supported to car view blockstores") } -func (uv *userView) PutMany(ctx context.Context, blks []blocks.Block) error { +func (uv *userView) PutMany(ctx context.Context, blks []blockformat.Block) error { return fmt.Errorf("puts not supported to car view blockstores") } @@ -222,54 +281,44 @@ func (uv *userView) GetSize(ctx context.Context, k cid.Cid) (int, error) { return len(blk.RawData()), nil } +// subset of blockstore.Blockstore that we actually use here +type minBlockstore interface { + Get(ctx context.Context, bcid cid.Cid) (blockformat.Block, error) + Has(ctx context.Context, bcid cid.Cid) (bool, error) + GetSize(ctx context.Context, bcid cid.Cid) (int, error) +} + type DeltaSession struct { - fresh blockstore.Blockstore - blks map[cid.Cid]blocks.Block - base blockstore.Blockstore - user uint + blks map[cid.Cid]blockformat.Block + rmcids map[cid.Cid]bool + base minBlockstore + user models.Uid + baseCid cid.Cid seq int readonly bool - cs *CarStore + cs shardWriter + lastRev string } -func (cs *CarStore) checkLastShardCache(user uint) *CarShard { - cs.lscLk.Lock() - defer cs.lscLk.Unlock() - - ls, ok := cs.lastShardCache[user] - if ok { - return ls - } - - return nil +func (cs *FileCarStore) checkLastShardCache(user models.Uid) *CarShard { + return cs.lastShardCache.check(user) } -func (cs *CarStore) putLastShardCache(user uint, ls *CarShard) { - cs.lscLk.Lock() - defer cs.lscLk.Unlock() - - cs.lastShardCache[user] = ls +func (cs *FileCarStore) removeLastShardCache(user models.Uid) { + cs.lastShardCache.remove(user) } -func (cs *CarStore) getLastShard(ctx context.Context, user uint) (*CarShard, error) { - maybeLs := cs.checkLastShardCache(user) - if maybeLs != nil { - return maybeLs, nil - } - - var lastShard CarShard - if err := cs.meta.WithContext(ctx).Model(CarShard{}).Limit(1).Order("id desc").Find(&lastShard, "usr = ?", user).Error; err != nil { - //if err := cs.meta.Model(CarShard{}).Where("user = ?", user).Last(&lastShard).Error; err != nil { - //if err != gorm.ErrRecordNotFound { - return nil, err - //} - } +func (cs *FileCarStore) putLastShardCache(ls *CarShard) { + cs.lastShardCache.put(ls) +} - cs.putLastShardCache(user, &lastShard) - return &lastShard, nil +func (cs *FileCarStore) getLastShard(ctx context.Context, user models.Uid) (*CarShard, error) { + return cs.lastShardCache.get(ctx, user) } -func (cs *CarStore) NewDeltaSession(ctx context.Context, user uint, prev *cid.Cid) (*DeltaSession, error) { +var ErrRepoBaseMismatch = fmt.Errorf("attempted a delta session on top of the wrong previous head") + +func (cs *FileCarStore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession") defer span.End() @@ -280,34 +329,33 @@ func (cs *CarStore) NewDeltaSession(ctx context.Context, user uint, prev *cid.Ci return nil, err } - if prev != nil { - if lastShard.Root != "" && lastShard.Root != prev.String() { - return nil, fmt.Errorf("attempted a delta session on top of the wrong previous head") - } + if since != nil && *since != lastShard.Rev { + return nil, fmt.Errorf("revision mismatch: %s != %s: %w", *since, lastShard.Rev, ErrRepoBaseMismatch) } return &DeltaSession{ - fresh: blockstore.NewBlockstore(datastore.NewMapDatastore()), - blks: make(map[cid.Cid]blocks.Block), + blks: make(map[cid.Cid]blockformat.Block), base: &userView{ user: user, - cs: cs, + cs: cs.meta, prefetch: true, - cache: make(map[cid.Cid]blocks.Block), + cache: make(map[cid.Cid]blockformat.Block), }, - user: user, - cs: cs, - seq: lastShard.Seq + 1, + user: user, + baseCid: lastShard.Root.CID, + cs: cs, + seq: lastShard.Seq + 1, + lastRev: lastShard.Rev, }, nil } -func (cs *CarStore) ReadOnlySession(user uint) (*DeltaSession, error) { +func (cs *FileCarStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { return &DeltaSession{ base: &userView{ user: user, - cs: cs, + cs: cs.meta, prefetch: false, - cache: make(map[cid.Cid]blocks.Block), + cache: make(map[cid.Cid]blockformat.Block), }, readonly: true, user: user, @@ -315,26 +363,27 @@ func (cs *CarStore) ReadOnlySession(user uint) (*DeltaSession, error) { }, nil } -func (cs *CarStore) ReadUserCar(ctx context.Context, user uint, until cid.Cid, incremental bool, w io.Writer) error { +// TODO: incremental is only ever called true, remove the param +func (cs *FileCarStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, shardOut io.Writer) error { ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") defer span.End() - var untilSeq int - - if until.Defined() { - var untilShard CarShard - if err := cs.meta.First(&untilShard, "root = ? AND usr = ?", until.String(), user).Error; err != nil { + var earlySeq int + if sinceRev != "" { + var err error + earlySeq, err = cs.meta.SeqForRev(ctx, user, sinceRev) + if err != nil { return err } - untilSeq = untilShard.Seq } - var shards []CarShard - if err := cs.meta.Order("seq desc").Find(&shards, "usr = ? AND seq >= ?", user, untilSeq).Error; err != nil { + shards, err := cs.meta.GetUserShardsDesc(ctx, user, earlySeq) + if err != nil { return err } - if !incremental && until.Defined() { + // TODO: incremental is only ever called true, so this is fine and we can remove the error check + if !incremental && earlySeq > 0 { // have to do it the ugly way return fmt.Errorf("nyi") } @@ -344,20 +393,15 @@ func (cs *CarStore) ReadUserCar(ctx context.Context, user uint, until cid.Cid, i } // fast path! - rootcid, err := cid.Decode(shards[0].Root) - if err != nil { - return err - } - if err := car.WriteHeader(&car.CarHeader{ - Roots: []cid.Cid{rootcid}, + Roots: []cid.Cid{shards[0].Root.CID}, Version: 1, - }, w); err != nil { + }, shardOut); err != nil { return err } for _, sh := range shards { - if err := cs.writeShardBlocks(ctx, &sh, w); err != nil { + if err := cs.writeShardBlocks(ctx, &sh, shardOut); err != nil { return err } } @@ -365,7 +409,9 @@ func (cs *CarStore) ReadUserCar(ctx context.Context, user uint, until cid.Cid, i return nil } -func (cs *CarStore) writeShardBlocks(ctx context.Context, sh *CarShard, w io.Writer) error { +// inner loop part of ReadUserCar +// copy shard blocks from disk to Writer +func (cs *FileCarStore) writeShardBlocks(ctx context.Context, sh *CarShard, shardOut io.Writer) error { ctx, span := otel.Tracer("carstore").Start(ctx, "writeShardBlocks") defer span.End() @@ -373,13 +419,14 @@ func (cs *CarStore) writeShardBlocks(ctx context.Context, sh *CarShard, w io.Wri if err != nil { return err } + defer fi.Close() _, err = fi.Seek(sh.DataStart, io.SeekStart) if err != nil { return err } - _, err = io.Copy(w, fi) + _, err = io.Copy(shardOut, fi) if err != nil { return err } @@ -387,9 +434,41 @@ func (cs *CarStore) writeShardBlocks(ctx context.Context, sh *CarShard, w io.Wri return nil } +// inner loop part of compactBucket +func (cs *FileCarStore) iterateShardBlocks(ctx context.Context, sh *CarShard, cb func(blk blockformat.Block) error) error { + fi, err := os.Open(sh.Path) + if err != nil { + return err + } + defer fi.Close() + + rr, err := car.NewCarReader(fi) + if err != nil { + return fmt.Errorf("opening shard car: %w", err) + } + + for { + blk, err := rr.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + + if err := cb(blk); err != nil { + return err + } + } +} + var _ blockstore.Blockstore = (*DeltaSession)(nil) -func (ds *DeltaSession) Put(ctx context.Context, b blocks.Block) error { +func (ds *DeltaSession) BaseCid() cid.Cid { + return ds.baseCid +} + +func (ds *DeltaSession) Put(ctx context.Context, b blockformat.Block) error { if ds.readonly { return fmt.Errorf("cannot write to readonly deltaSession") } @@ -397,7 +476,7 @@ func (ds *DeltaSession) Put(ctx context.Context, b blocks.Block) error { return nil } -func (ds *DeltaSession) PutMany(ctx context.Context, bs []blocks.Block) error { +func (ds *DeltaSession) PutMany(ctx context.Context, bs []blockformat.Block) error { if ds.readonly { return fmt.Errorf("cannot write to readonly deltaSession") } @@ -418,14 +497,14 @@ func (ds *DeltaSession) DeleteBlock(ctx context.Context, c cid.Cid) error { } if _, ok := ds.blks[c]; !ok { - return ipld.ErrNotFound{c} + return ipld.ErrNotFound{Cid: c} } delete(ds.blks, c) return nil } -func (ds *DeltaSession) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { +func (ds *DeltaSession) Get(ctx context.Context, c cid.Cid) (blockformat.Block, error) { b, ok := ds.blks[c] if ok { return b, nil @@ -456,9 +535,17 @@ func (ds *DeltaSession) GetSize(ctx context.Context, c cid.Cid) (int, error) { return ds.base.GetSize(ctx, c) } -func (cs *CarStore) openNewShardFile(ctx context.Context, user uint, seq int) (*os.File, string, error) { +func fnameForShard(user models.Uid, seq int) string { + return fmt.Sprintf("sh-%d-%d", user, seq) +} + +func (cs *FileCarStore) dirForUser(user models.Uid) string { + return cs.rootDirs[int(user)%len(cs.rootDirs)] +} + +func (cs *FileCarStore) openNewShardFile(ctx context.Context, user models.Uid, seq int) (*os.File, string, error) { // TODO: some overwrite protections - fname := filepath.Join(cs.rootDir, fmt.Sprintf("sh-%d-%d", user, seq)) + fname := filepath.Join(cs.dirForUser(user), fnameForShard(user, seq)) fi, err := os.Create(fname) if err != nil { return nil, "", err @@ -467,9 +554,12 @@ func (cs *CarStore) openNewShardFile(ctx context.Context, user uint, seq int) (* return fi, fname, nil } -func (cs *CarStore) writeNewShardFile(ctx context.Context, user uint, seq int, data []byte) (string, error) { +func (cs *FileCarStore) writeNewShardFile(ctx context.Context, user models.Uid, seq int, data []byte) (string, error) { + _, span := otel.Tracer("carstore").Start(ctx, "writeNewShardFile") + defer span.End() + // TODO: some overwrite protections - fname := filepath.Join(cs.rootDir, fmt.Sprintf("sh-%d-%d", user, seq)) + fname := filepath.Join(cs.dirForUser(user), fnameForShard(user, seq)) if err := os.WriteFile(fname, data, 0664); err != nil { return "", err } @@ -477,9 +567,13 @@ func (cs *CarStore) writeNewShardFile(ctx context.Context, user uint, seq int, d return fname, nil } +func (cs *FileCarStore) deleteShardFile(ctx context.Context, sh *CarShard) error { + return os.Remove(sh.Path) +} + // CloseWithRoot writes all new blocks in a car file to the writer with the // given cid as the 'root' -func (ds *DeltaSession) CloseWithRoot(ctx context.Context, root cid.Cid) ([]byte, error) { +func (ds *DeltaSession) CloseWithRoot(ctx context.Context, root cid.Cid, rev string) ([]byte, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "CloseWithRoot") defer span.End() @@ -487,28 +581,69 @@ func (ds *DeltaSession) CloseWithRoot(ctx context.Context, root cid.Cid) ([]byte return nil, fmt.Errorf("cannot write to readonly deltaSession") } - buf := new(bytes.Buffer) + return ds.cs.writeNewShard(ctx, root, rev, ds.user, ds.seq, ds.blks, ds.rmcids) +} + +func WriteCarHeader(w io.Writer, root cid.Cid) (int64, error) { h := &car.CarHeader{ Roots: []cid.Cid{root}, Version: 1, } hb, err := cbor.DumpObject(h) if err != nil { - return nil, err + return 0, err } - hnw, err := LdWrite(buf, hb) + hnw, err := LdWrite(w, hb) if err != nil { - return nil, err + return 0, err + } + + return hnw, nil +} + +// shardWriter.writeNewShard called from inside DeltaSession.CloseWithRoot +type shardWriter interface { + // writeNewShard stores blocks in `blks` arg and creates a new shard to propagate out to our firehose + writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) +} + +func blocksToCar(ctx context.Context, root cid.Cid, rev string, blks map[cid.Cid]blockformat.Block) ([]byte, error) { + buf := new(bytes.Buffer) + _, err := WriteCarHeader(buf, root) + if err != nil { + return nil, fmt.Errorf("failed to write car header: %w", err) + } + + for k, blk := range blks { + _, err := LdWrite(buf, k.Bytes(), blk.RawData()) + if err != nil { + return nil, fmt.Errorf("failed to write block: %w", err) + } } - var offset int64 = hnw + return buf.Bytes(), nil +} + +func (cs *FileCarStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { + + buf := new(bytes.Buffer) + hnw, err := WriteCarHeader(buf, root) + if err != nil { + return nil, fmt.Errorf("failed to write car header: %w", err) + } + + // TODO: writing these blocks in map traversal order is bad, I believe the + // optimal ordering will be something like reverse-write-order, but random + // is definitely not it + + offset := hnw //brefs := make([]*blockRef, 0, len(ds.blks)) - brefs := make([]map[string]interface{}, 0, len(ds.blks)) - for k, blk := range ds.blks { + brefs := make([]map[string]interface{}, 0, len(blks)) + for k, blk := range blks { nw, err := LdWrite(buf, k.Bytes(), blk.RawData()) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to write block: %w", err) } /* @@ -521,70 +656,122 @@ func (ds *DeltaSession) CloseWithRoot(ctx context.Context, root cid.Cid) ([]byte // adding things to the db by map is the only way to get gorm to not // add the 'returning' clause, which costs a lot of time brefs = append(brefs, map[string]interface{}{ - "cid": k.String(), + "cid": models.DbCID{CID: k}, "offset": offset, }) offset += nw } - path, err := ds.cs.writeNewShardFile(ctx, ds.user, ds.seq, buf.Bytes()) + start := time.Now() + path, err := cs.writeNewShardFile(ctx, user, seq, buf.Bytes()) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to write shard file: %w", err) } + writeShardFileDuration.Observe(time.Since(start).Seconds()) - // TODO: all this database work needs to be in a single transaction shard := CarShard{ - Root: root.String(), + Root: models.DbCID{CID: root}, DataStart: hnw, - Seq: ds.seq, + Seq: seq, Path: path, - Usr: ds.user, + Usr: user, + Rev: rev, } - // TODO: there should be a way to create the shard and block_refs that - // reference it in the same query, would save a lot of time - if err := ds.cs.meta.WithContext(ctx).Create(&shard).Error; err != nil { + start = time.Now() + if err := cs.putShard(ctx, &shard, brefs, rmcids, false); err != nil { return nil, err } - ds.cs.putLastShardCache(ds.user, &shard) + writeShardMetadataDuration.Observe(time.Since(start).Seconds()) + + return buf.Bytes(), nil +} - for _, ref := range brefs { - ref["shard"] = shard.ID +func (cs *FileCarStore) putShard(ctx context.Context, shard *CarShard, brefs []map[string]any, rmcids map[cid.Cid]bool, nocache bool) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "putShard") + defer span.End() + + err := cs.meta.PutShardAndRefs(ctx, shard, brefs, rmcids) + if err != nil { + return err } - if err := ds.cs.meta.WithContext(ctx).Table("block_refs").Create(brefs).Error; err != nil { - return nil, err + if !nocache { + cs.putLastShardCache(shard) } - return buf.Bytes(), nil + return nil } -func LdWrite(w io.Writer, d ...[]byte) (int64, error) { - var sum uint64 - for _, s := range d { - sum += uint64(len(s)) +func BlockDiff(ctx context.Context, bs blockstore.Blockstore, oldroot cid.Cid, newcids map[cid.Cid]blockformat.Block, skipcids map[cid.Cid]bool) (map[cid.Cid]bool, error) { + ctx, span := otel.Tracer("repo").Start(ctx, "BlockDiff") + defer span.End() + + if !oldroot.Defined() { + return map[cid.Cid]bool{}, nil } - buf := make([]byte, 8) - n := binary.PutUvarint(buf, sum) - nw, err := w.Write(buf[:n]) - if err != nil { - return 0, err + // walk the entire 'new' portion of the tree, marking all referenced cids as 'keep' + keepset := make(map[cid.Cid]bool) + for c := range newcids { + keepset[c] = true + oblk, err := bs.Get(ctx, c) + if err != nil { + return nil, fmt.Errorf("get failed in new tree: %w", err) + } + + if err := cbg.ScanForLinks(bytes.NewReader(oblk.RawData()), func(lnk cid.Cid) { + keepset[lnk] = true + }); err != nil { + return nil, err + } } - for _, s := range d { - onw, err := w.Write(s) + if keepset[oldroot] { + // this should probably never happen, but is technically correct + return nil, nil + } + + // next, walk the old tree from the root, recursing on cids *not* in the keepset. + dropset := make(map[cid.Cid]bool) + dropset[oldroot] = true + queue := []cid.Cid{oldroot} + + for len(queue) > 0 { + c := queue[0] + queue = queue[1:] + + if skipcids != nil && skipcids[c] { + continue + } + + oblk, err := bs.Get(ctx, c) if err != nil { - return int64(nw), err + return nil, fmt.Errorf("get failed in old tree: %w", err) + } + + if err := cbg.ScanForLinks(bytes.NewReader(oblk.RawData()), func(lnk cid.Cid) { + if lnk.Prefix().Codec != cid.DagCBOR { + return + } + + if !keepset[lnk] { + dropset[lnk] = true + queue = append(queue, lnk) + } + }); err != nil { + return nil, err } - nw += onw } - return int64(nw), nil + return dropset, nil } -func (cs *CarStore) ImportSlice(ctx context.Context, uid uint, carslice []byte) (cid.Cid, *DeltaSession, error) { +func (cs *FileCarStore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "ImportSlice") + defer span.End() + carr, err := car.NewCarReader(bytes.NewReader(carslice)) if err != nil { return cid.Undef, nil, err @@ -594,11 +781,12 @@ func (cs *CarStore) ImportSlice(ctx context.Context, uid uint, carslice []byte) return cid.Undef, nil, fmt.Errorf("invalid car file, header must have a single root (has %d)", len(carr.Header.Roots)) } - ds, err := cs.NewDeltaSession(ctx, uid, nil) + ds, err := cs.NewDeltaSession(ctx, uid, since) if err != nil { - return cid.Undef, nil, err + return cid.Undef, nil, fmt.Errorf("new delta session failed: %w", err) } + var cids []cid.Cid for { blk, err := carr.Next() if err != nil { @@ -608,6 +796,8 @@ func (cs *CarStore) ImportSlice(ctx context.Context, uid uint, carslice []byte) return cid.Undef, nil, err } + cids = append(cids, blk.Cid()) + if err := ds.Put(ctx, blk); err != nil { return cid.Undef, nil, err } @@ -615,3 +805,591 @@ func (cs *CarStore) ImportSlice(ctx context.Context, uid uint, carslice []byte) return carr.Header.Roots[0], ds, nil } + +func (ds *DeltaSession) CalcDiff(ctx context.Context, skipcids map[cid.Cid]bool) error { + rmcids, err := BlockDiff(ctx, ds, ds.baseCid, ds.blks, skipcids) + if err != nil { + return fmt.Errorf("block diff failed (base=%s,rev=%s): %w", ds.baseCid, ds.lastRev, err) + } + + ds.rmcids = rmcids + return nil +} + +func (cs *FileCarStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { + lastShard, err := cs.getLastShard(ctx, user) + if err != nil { + return cid.Undef, err + } + if lastShard.ID == 0 { + return cid.Undef, nil + } + + return lastShard.Root.CID, nil +} + +func (cs *FileCarStore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { + lastShard, err := cs.getLastShard(ctx, user) + if err != nil { + return "", err + } + if lastShard.ID == 0 { + return "", nil + } + + return lastShard.Rev, nil +} + +type UserStat struct { + Seq int + Root string + Created time.Time +} + +func (cs *FileCarStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { + shards, err := cs.meta.GetUserShards(ctx, usr) + if err != nil { + return nil, err + } + + var out []UserStat + for _, s := range shards { + out = append(out, UserStat{ + Seq: s.Seq, + Root: s.Root.CID.String(), + Created: s.CreatedAt, + }) + } + + return out, nil +} + +func (cs *FileCarStore) WipeUserData(ctx context.Context, user models.Uid) error { + shards, err := cs.meta.GetUserShards(ctx, user) + if err != nil { + return err + } + + if err := cs.deleteShards(ctx, shards); err != nil { + if !os.IsNotExist(err) { + return err + } + } + + cs.removeLastShardCache(user) + + return nil +} + +func (cs *FileCarStore) deleteShards(ctx context.Context, shs []CarShard) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "deleteShards") + defer span.End() + + deleteSlice := func(ctx context.Context, subs []CarShard) error { + ids := make([]uint, len(subs)) + for i, sh := range subs { + ids[i] = sh.ID + } + + err := cs.meta.DeleteShardsAndRefs(ctx, ids) + if err != nil { + return err + } + + for _, sh := range subs { + if err := cs.deleteShardFile(ctx, &sh); err != nil { + if !os.IsNotExist(err) { + return err + } + cs.log.Warn("shard file we tried to delete did not exist", "shard", sh.ID, "path", sh.Path) + } + } + + return nil + } + + chunkSize := 2000 + for i := 0; i < len(shs); i += chunkSize { + sl := shs[i:] + if len(sl) > chunkSize { + sl = sl[:chunkSize] + } + + if err := deleteSlice(ctx, sl); err != nil { + return err + } + } + + return nil +} + +type shardStat struct { + ID uint + Dirty int + Seq int + Total int + + refs []blockRef +} + +func (s shardStat) dirtyFrac() float64 { + return float64(s.Dirty) / float64(s.Total) +} + +func aggrRefs(brefs []blockRef, shards map[uint]CarShard, staleCids map[cid.Cid]bool) []shardStat { + byId := make(map[uint]*shardStat) + + for _, br := range brefs { + s, ok := byId[br.Shard] + if !ok { + s = &shardStat{ + ID: br.Shard, + Seq: shards[br.Shard].Seq, + } + byId[br.Shard] = s + } + + s.Total++ + if staleCids[br.Cid.CID] { + s.Dirty++ + } + + s.refs = append(s.refs, br) + } + + var out []shardStat + for _, s := range byId { + out = append(out, *s) + } + + sort.Slice(out, func(i, j int) bool { + return out[i].Seq < out[j].Seq + }) + + return out +} + +type compBucket struct { + shards []shardStat + + cleanBlocks int + expSize int +} + +func (cb *compBucket) shouldCompact() bool { + if len(cb.shards) == 0 { + return false + } + + if len(cb.shards) > 5 { + return true + } + + var frac float64 + for _, s := range cb.shards { + frac += s.dirtyFrac() + } + frac /= float64(len(cb.shards)) + + if len(cb.shards) > 3 && frac > 0.2 { + return true + } + + return frac > 0.4 +} + +func (cb *compBucket) addShardStat(ss shardStat) { + cb.cleanBlocks += (ss.Total - ss.Dirty) + cb.shards = append(cb.shards, ss) +} + +func (cb *compBucket) isEmpty() bool { + return len(cb.shards) == 0 +} + +func (cs *FileCarStore) openNewCompactedShardFile(ctx context.Context, user models.Uid, seq int) (*os.File, string, error) { + // TODO: some overwrite protections + // NOTE CreateTemp is used for creating a non-colliding file, but we keep it and don't delete it so don't think of it as "temporary". + // This creates "sh-%d-%d%s" with some random stuff in the last position + fi, err := os.CreateTemp(cs.dirForUser(user), fnameForShard(user, seq)) + if err != nil { + return nil, "", err + } + + return fi, fi.Name(), nil +} + +type CompactionTarget struct { + Usr models.Uid + NumShards int +} + +func (cs *FileCarStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "GetCompactionTargets") + defer span.End() + + return cs.meta.GetCompactionTargets(ctx, shardCount) +} + +// getBlockRefsForShards is a prep function for CompactUserShards +func (cs *FileCarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) ([]blockRef, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "getBlockRefsForShards") + defer span.End() + + span.SetAttributes(attribute.Int("shards", len(shardIds))) + + out, err := cs.meta.GetBlockRefsForShards(ctx, shardIds) + if err != nil { + return nil, err + } + + span.SetAttributes(attribute.Int("refs", len(out))) + + return out, nil +} + +func shardSize(sh *CarShard) (int64, error) { + st, err := os.Stat(sh.Path) + if err != nil { + if os.IsNotExist(err) { + slog.Warn("missing shard, return size of zero", "path", sh.Path, "shard", sh.ID, "system", "carstore") + return 0, nil + } + return 0, fmt.Errorf("stat %q: %w", sh.Path, err) + } + + return st.Size(), nil +} + +type CompactionStats struct { + TotalRefs int `json:"totalRefs"` + StartShards int `json:"startShards"` + NewShards int `json:"newShards"` + SkippedShards int `json:"skippedShards"` + ShardsDeleted int `json:"shardsDeleted"` + DupeCount int `json:"dupeCount"` +} + +func (cs *FileCarStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "CompactUserShards") + defer span.End() + + span.SetAttributes(attribute.Int64("user", int64(user))) + + shards, err := cs.meta.GetUserShards(ctx, user) + if err != nil { + return nil, err + } + + if skipBigShards { + // Since we generally expect shards to start bigger and get smaller, + // and because we want to avoid compacting non-adjacent shards + // together, and because we want to avoid running a stat on every + // single shard (can be expensive for repos that haven't been compacted + // in a while) we only skip a prefix of shard files that are over the + // threshold. this may end up not skipping some shards that are over + // the threshold if a below-threshold shard occurs before them, but + // since this is a heuristic and imperfect optimization, that is + // acceptable. + var skip int + for i, sh := range shards { + size, err := shardSize(&sh) + if err != nil { + return nil, fmt.Errorf("could not check size of shard file: %w", err) + } + + if size > BigShardThreshold { + skip = i + 1 + } else { + break + } + } + shards = shards[skip:] + } + + span.SetAttributes(attribute.Int("shards", len(shards))) + + var shardIds []uint + for _, s := range shards { + shardIds = append(shardIds, s.ID) + } + + shardsById := make(map[uint]CarShard) + for _, s := range shards { + shardsById[s.ID] = s + } + + brefs, err := cs.getBlockRefsForShards(ctx, shardIds) + if err != nil { + return nil, fmt.Errorf("getting block refs failed: %w", err) + } + + span.SetAttributes(attribute.Int("blockRefs", len(brefs))) + + staleRefs, err := cs.meta.GetUserStaleRefs(ctx, user) + if err != nil { + return nil, err + } + + span.SetAttributes(attribute.Int("staleRefs", len(staleRefs))) + + stale := make(map[cid.Cid]bool) + for _, br := range staleRefs { + cids, err := br.getCids() + if err != nil { + return nil, fmt.Errorf("failed to unpack cids from staleRefs record (%d): %w", br.ID, err) + } + for _, c := range cids { + stale[c] = true + } + } + + // if we have a staleRef that references multiple blockRefs, we consider that block a 'dirty duplicate' + var dupes []cid.Cid + var hasDirtyDupes bool + seenBlocks := make(map[cid.Cid]bool) + for _, br := range brefs { + if seenBlocks[br.Cid.CID] { + dupes = append(dupes, br.Cid.CID) + hasDirtyDupes = true + delete(stale, br.Cid.CID) + } else { + seenBlocks[br.Cid.CID] = true + } + } + + for _, dupe := range dupes { + delete(stale, dupe) // remove dupes from stale list, see comment below + } + + if hasDirtyDupes { + // if we have no duplicates, then the keep set is simply all the 'clean' blockRefs + // in the case we have duplicate dirty references we have to compute + // the keep set by walking the entire repo to check if anything is + // still referencing the dirty block in question + + // we could also just add the duplicates to the keep set for now and + // focus on compacting everything else. it leaves *some* dirty blocks + // still around but we're doing that anyways since compaction isn't a + // perfect process + + cs.log.Debug("repo has dirty dupes", "count", len(dupes), "uid", user, "staleRefs", len(staleRefs), "blockRefs", len(brefs)) + + //return nil, fmt.Errorf("WIP: not currently handling this case") + } + + keep := make(map[cid.Cid]bool) + for _, br := range brefs { + if !stale[br.Cid.CID] { + keep[br.Cid.CID] = true + } + } + + for _, dupe := range dupes { + keep[dupe] = true + } + + results := aggrRefs(brefs, shardsById, stale) + var sum int + for _, r := range results { + sum += r.Total + } + + lowBound := 20 + N := 10 + // we want to *aim* for N shards per user + // the last several should be left small to allow easy loading from disk + // for updates (since recent blocks are most likely needed for edits) + // the beginning of the list should be some sort of exponential fall-off + // with the area under the curve targeted by the total number of blocks we + // have + var threshs []int + tot := len(brefs) + for i := 0; i < N; i++ { + v := tot / 2 + if v < lowBound { + v = lowBound + } + tot = tot / 2 + threshs = append(threshs, v) + } + + thresholdForPosition := func(i int) int { + if i >= len(threshs) { + return lowBound + } + return threshs[i] + } + + cur := new(compBucket) + cur.expSize = thresholdForPosition(0) + var compactionQueue []*compBucket + for i, r := range results { + cur.addShardStat(r) + + if cur.cleanBlocks > cur.expSize || i > len(results)-3 { + compactionQueue = append(compactionQueue, cur) + cur = &compBucket{ + expSize: thresholdForPosition(len(compactionQueue)), + } + } + } + if !cur.isEmpty() { + compactionQueue = append(compactionQueue, cur) + } + + stats := &CompactionStats{ + StartShards: len(shards), + TotalRefs: len(brefs), + } + + removedShards := make(map[uint]bool) + for _, b := range compactionQueue { + if !b.shouldCompact() { + stats.SkippedShards += len(b.shards) + continue + } + + if err := cs.compactBucket(ctx, user, b, shardsById, keep); err != nil { + return nil, fmt.Errorf("compact bucket: %w", err) + } + + stats.NewShards++ + + todelete := make([]CarShard, 0, len(b.shards)) + for _, s := range b.shards { + removedShards[s.ID] = true + sh, ok := shardsById[s.ID] + if !ok { + return nil, fmt.Errorf("missing shard to delete") + } + + todelete = append(todelete, sh) + } + + stats.ShardsDeleted += len(todelete) + if err := cs.deleteShards(ctx, todelete); err != nil { + return nil, fmt.Errorf("deleting shards: %w", err) + } + } + + // now we need to delete the staleRefs we successfully cleaned up + // we can safely delete a staleRef if all the shards that have blockRefs with matching stale refs were processed + if err := cs.deleteStaleRefs(ctx, user, brefs, staleRefs, removedShards); err != nil { + return nil, fmt.Errorf("delete stale refs: %w", err) + } + + stats.DupeCount = len(dupes) + + return stats, nil +} + +func (cs *FileCarStore) deleteStaleRefs(ctx context.Context, uid models.Uid, brefs []blockRef, staleRefs []staleRef, removedShards map[uint]bool) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "deleteStaleRefs") + defer span.End() + + brByCid := make(map[cid.Cid][]blockRef) + for _, br := range brefs { + brByCid[br.Cid.CID] = append(brByCid[br.Cid.CID], br) + } + + var staleToKeep []cid.Cid + for _, sr := range staleRefs { + cids, err := sr.getCids() + if err != nil { + return fmt.Errorf("getCids on staleRef failed (%d): %w", sr.ID, err) + } + + for _, c := range cids { + brs := brByCid[c] + del := true + for _, br := range brs { + if !removedShards[br.Shard] { + del = false + break + } + } + + if !del { + staleToKeep = append(staleToKeep, c) + } + } + } + + return cs.meta.SetStaleRef(ctx, uid, staleToKeep) +} + +func (cs *FileCarStore) compactBucket(ctx context.Context, user models.Uid, b *compBucket, shardsById map[uint]CarShard, keep map[cid.Cid]bool) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "compactBucket") + defer span.End() + + span.SetAttributes(attribute.Int("shards", len(b.shards))) + + last := b.shards[len(b.shards)-1] + lastsh := shardsById[last.ID] + fi, path, err := cs.openNewCompactedShardFile(ctx, user, last.Seq) + if err != nil { + return fmt.Errorf("opening new file: %w", err) + } + + defer fi.Close() + root := lastsh.Root.CID + + hnw, err := WriteCarHeader(fi, root) + if err != nil { + return err + } + + offset := hnw + var nbrefs []map[string]any + written := make(map[cid.Cid]bool) + for _, s := range b.shards { + sh := shardsById[s.ID] + if err := cs.iterateShardBlocks(ctx, &sh, func(blk blockformat.Block) error { + if written[blk.Cid()] { + return nil + } + + if keep[blk.Cid()] { + nw, err := LdWrite(fi, blk.Cid().Bytes(), blk.RawData()) + if err != nil { + return fmt.Errorf("failed to write block: %w", err) + } + + nbrefs = append(nbrefs, map[string]interface{}{ + "cid": models.DbCID{CID: blk.Cid()}, + "offset": offset, + }) + + offset += nw + written[blk.Cid()] = true + } + return nil + }); err != nil { + // If we ever fail to iterate a shard file because its + // corrupted, just log an error and skip the shard + cs.log.Error("iterating blocks in shard", "shard", s.ID, "err", err, "uid", user) + } + } + + shard := CarShard{ + Root: models.DbCID{CID: root}, + DataStart: hnw, + Seq: lastsh.Seq, + Path: path, + Usr: user, + Rev: lastsh.Rev, + } + + if err := cs.putShard(ctx, &shard, nbrefs, nil, true); err != nil { + // if writing the shard fails, we should also delete the file + _ = fi.Close() + + if err2 := os.Remove(fi.Name()); err2 != nil { + cs.log.Error("failed to remove shard file after failed db transaction", "path", fi.Name(), "err", err2) + } + + return err + } + return nil +} diff --git a/carstore/last_shard_cache.go b/carstore/last_shard_cache.go new file mode 100644 index 000000000..8371b8883 --- /dev/null +++ b/carstore/last_shard_cache.go @@ -0,0 +1,70 @@ +package carstore + +import ( + "context" + "github.com/bluesky-social/indigo/models" + "go.opentelemetry.io/otel" + "sync" +) + +type LastShardSource interface { + GetLastShard(context.Context, models.Uid) (*CarShard, error) +} + +type lastShardCache struct { + source LastShardSource + + lscLk sync.Mutex + lastShardCache map[models.Uid]*CarShard +} + +func (lsc *lastShardCache) Init() { + lsc.lastShardCache = make(map[models.Uid]*CarShard) +} + +func (lsc *lastShardCache) check(user models.Uid) *CarShard { + lsc.lscLk.Lock() + defer lsc.lscLk.Unlock() + + ls, ok := lsc.lastShardCache[user] + if ok { + return ls + } + + return nil +} + +func (lsc *lastShardCache) remove(user models.Uid) { + lsc.lscLk.Lock() + defer lsc.lscLk.Unlock() + + delete(lsc.lastShardCache, user) +} + +func (lsc *lastShardCache) put(ls *CarShard) { + if ls == nil { + return + } + lsc.lscLk.Lock() + defer lsc.lscLk.Unlock() + + lsc.lastShardCache[ls.Usr] = ls +} + +func (lsc *lastShardCache) get(ctx context.Context, user models.Uid) (*CarShard, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "getLastShard") + defer span.End() + + maybeLs := lsc.check(user) + if maybeLs != nil { + return maybeLs, nil + } + + lastShard, err := lsc.source.GetLastShard(ctx, user) + if err != nil { + return nil, err + } + + lsc.put(lastShard) + return lastShard, nil +} diff --git a/carstore/meta_gorm.go b/carstore/meta_gorm.go new file mode 100644 index 000000000..eb9ff7bbc --- /dev/null +++ b/carstore/meta_gorm.go @@ -0,0 +1,346 @@ +package carstore + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/bluesky-social/indigo/models" + "github.com/ipfs/go-cid" + "go.opentelemetry.io/otel" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +type CarStoreGormMeta struct { + meta *gorm.DB +} + +func (cs *CarStoreGormMeta) Init() error { + if err := cs.meta.AutoMigrate(&CarShard{}, &blockRef{}); err != nil { + return err + } + if err := cs.meta.AutoMigrate(&staleRef{}); err != nil { + return err + } + return nil +} + +// Return true if any known record matches (Uid, Cid) +func (cs *CarStoreGormMeta) HasUidCid(ctx context.Context, user models.Uid, k cid.Cid) (bool, error) { + var count int64 + if err := cs.meta. + Model(blockRef{}). + Select("path, block_refs.offset"). + Joins("left join car_shards on block_refs.shard = car_shards.id"). + Where("usr = ? AND cid = ?", user, models.DbCID{CID: k}). + Count(&count).Error; err != nil { + return false, err + } + + return count > 0, nil +} + +// For some Cid, lookup the block ref. +// Return the path of the file written, the offset within the file, and the user associated with the Cid. +func (cs *CarStoreGormMeta) LookupBlockRef(ctx context.Context, k cid.Cid) (path string, offset int64, user models.Uid, err error) { + // TODO: for now, im using a join to ensure we only query blocks from the + // correct user. maybe it makes sense to put the user in the blockRef + // directly? tradeoff of time vs space + var info struct { + Path string + Offset int64 + Usr models.Uid + } + if err := cs.meta.Raw(`SELECT + (select path from car_shards where id = block_refs.shard) as path, + block_refs.offset, + (select usr from car_shards where id = block_refs.shard) as usr +FROM block_refs +WHERE + block_refs.cid = ? +LIMIT 1;`, models.DbCID{CID: k}).Scan(&info).Error; err != nil { + var defaultUser models.Uid + return "", -1, defaultUser, err + } + return info.Path, info.Offset, info.Usr, nil +} + +func (cs *CarStoreGormMeta) GetLastShard(ctx context.Context, user models.Uid) (*CarShard, error) { + var lastShard CarShard + if err := cs.meta.WithContext(ctx).Model(CarShard{}).Limit(1).Order("seq desc").Find(&lastShard, "usr = ?", user).Error; err != nil { + return nil, err + } + return &lastShard, nil +} + +// return all of a users's shards, ascending by Seq +func (cs *CarStoreGormMeta) GetUserShards(ctx context.Context, usr models.Uid) ([]CarShard, error) { + var shards []CarShard + if err := cs.meta.Order("seq asc").Find(&shards, "usr = ?", usr).Error; err != nil { + return nil, err + } + return shards, nil +} + +// return all of a users's shards, descending by Seq +func (cs *CarStoreGormMeta) GetUserShardsDesc(ctx context.Context, usr models.Uid, minSeq int) ([]CarShard, error) { + var shards []CarShard + if err := cs.meta.Order("seq desc").Find(&shards, "usr = ? AND seq >= ?", usr, minSeq).Error; err != nil { + return nil, err + } + return shards, nil +} + +func (cs *CarStoreGormMeta) GetUserStaleRefs(ctx context.Context, user models.Uid) ([]staleRef, error) { + var staleRefs []staleRef + if err := cs.meta.WithContext(ctx).Find(&staleRefs, "usr = ?", user).Error; err != nil { + return nil, err + } + return staleRefs, nil +} + +func (cs *CarStoreGormMeta) SeqForRev(ctx context.Context, user models.Uid, sinceRev string) (int, error) { + var untilShard CarShard + if err := cs.meta.Where("rev >= ? AND usr = ?", sinceRev, user).Order("rev").First(&untilShard).Error; err != nil { + return 0, fmt.Errorf("finding early shard: %w", err) + } + return untilShard.Seq, nil +} + +func (cs *CarStoreGormMeta) GetCompactionTargets(ctx context.Context, minShardCount int) ([]CompactionTarget, error) { + var targets []CompactionTarget + if err := cs.meta.Raw(`select usr, count(*) as num_shards from car_shards group by usr having count(*) > ? order by num_shards desc`, minShardCount).Scan(&targets).Error; err != nil { + return nil, err + } + + return targets, nil +} + +func (cs *CarStoreGormMeta) PutShardAndRefs(ctx context.Context, shard *CarShard, brefs []map[string]any, rmcids map[cid.Cid]bool) error { + // TODO: there should be a way to create the shard and block_refs that + // reference it in the same query, would save a lot of time + tx := cs.meta.WithContext(ctx).Begin() + + if err := tx.WithContext(ctx).Create(shard).Error; err != nil { + return fmt.Errorf("failed to create shard in DB tx: %w", err) + } + + for _, ref := range brefs { + ref["shard"] = shard.ID + } + + if err := createBlockRefs(ctx, tx, brefs); err != nil { + return fmt.Errorf("failed to create block refs: %w", err) + } + + if len(rmcids) > 0 { + cids := make([]cid.Cid, 0, len(rmcids)) + for c := range rmcids { + cids = append(cids, c) + } + + if err := tx.Create(&staleRef{ + Cids: packCids(cids), + Usr: shard.Usr, + }).Error; err != nil { + return err + } + } + + err := tx.WithContext(ctx).Commit().Error + if err != nil { + return fmt.Errorf("failed to commit shard DB transaction: %w", err) + } + return nil +} + +func (cs *CarStoreGormMeta) DeleteShardsAndRefs(ctx context.Context, ids []uint) error { + txn := cs.meta.Begin() + + if err := txn.Delete(&CarShard{}, "id in (?)", ids).Error; err != nil { + txn.Rollback() + return err + } + + if err := txn.Delete(&blockRef{}, "shard in (?)", ids).Error; err != nil { + txn.Rollback() + return err + } + + return txn.Commit().Error +} + +func (cs *CarStoreGormMeta) GetBlockRefsForShards(ctx context.Context, shardIds []uint) ([]blockRef, error) { + chunkSize := 2000 + out := make([]blockRef, 0, len(shardIds)) + for i := 0; i < len(shardIds); i += chunkSize { + sl := shardIds[i:] + if len(sl) > chunkSize { + sl = sl[:chunkSize] + } + + if err := blockRefsForShards(ctx, cs.meta, sl, &out); err != nil { + return nil, fmt.Errorf("getting block refs: %w", err) + } + } + return out, nil +} + +// blockRefsForShards is an inner loop helper for GetBlockRefsForShards +func blockRefsForShards(ctx context.Context, db *gorm.DB, shards []uint, obuf *[]blockRef) error { + // Check the database driver + switch db.Dialector.(type) { + case *postgres.Dialector: + sval := valuesStatementForShards(shards) + q := fmt.Sprintf(`SELECT block_refs.* FROM block_refs INNER JOIN (VALUES %s) AS vals(v) ON block_refs.shard = v`, sval) + return db.Raw(q).Scan(obuf).Error + default: + return db.Raw(`SELECT * FROM block_refs WHERE shard IN (?)`, shards).Scan(obuf).Error + } +} + +// valuesStatementForShards builds a postgres compatible statement string from int literals +func valuesStatementForShards(shards []uint) string { + sb := new(strings.Builder) + for i, v := range shards { + sb.WriteByte('(') + sb.WriteString(strconv.Itoa(int(v))) + sb.WriteByte(')') + if i != len(shards)-1 { + sb.WriteByte(',') + } + } + return sb.String() +} + +func (cs *CarStoreGormMeta) SetStaleRef(ctx context.Context, uid models.Uid, staleToKeep []cid.Cid) error { + txn := cs.meta.Begin() + + if err := txn.Delete(&staleRef{}, "usr = ?", uid).Error; err != nil { + return err + } + + // now create a new staleRef with all the refs we couldn't clear out + if len(staleToKeep) > 0 { + if err := txn.Create(&staleRef{ + Usr: uid, + Cids: packCids(staleToKeep), + }).Error; err != nil { + return err + } + } + + if err := txn.Commit().Error; err != nil { + return fmt.Errorf("failed to commit staleRef updates: %w", err) + } + return nil +} + +type CarShard struct { + ID uint `gorm:"primarykey"` + CreatedAt time.Time + + Root models.DbCID `gorm:"index"` + DataStart int64 + Seq int `gorm:"index:idx_car_shards_seq;index:idx_car_shards_usr_seq,priority:2,sort:desc"` + Path string + Usr models.Uid `gorm:"index:idx_car_shards_usr;index:idx_car_shards_usr_seq,priority:1"` + Rev string +} + +type blockRef struct { + ID uint `gorm:"primarykey"` + Cid models.DbCID `gorm:"index"` + Shard uint `gorm:"index"` + Offset int64 + //User uint `gorm:"index"` +} + +type staleRef struct { + ID uint `gorm:"primarykey"` + Cid *models.DbCID + Cids []byte + Usr models.Uid `gorm:"index"` +} + +func (sr *staleRef) getCids() ([]cid.Cid, error) { + if sr.Cid != nil { + return []cid.Cid{sr.Cid.CID}, nil + } + + return unpackCids(sr.Cids) +} + +func unpackCids(b []byte) ([]cid.Cid, error) { + br := bytes.NewReader(b) + var out []cid.Cid + for { + _, c, err := cid.CidFromReader(br) + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + out = append(out, c) + } + + return out, nil +} + +func packCids(cids []cid.Cid) []byte { + buf := new(bytes.Buffer) + for _, c := range cids { + buf.Write(c.Bytes()) + } + + return buf.Bytes() +} + +func createBlockRefs(ctx context.Context, tx *gorm.DB, brefs []map[string]any) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "createBlockRefs") + defer span.End() + + if err := createInBatches(ctx, tx, brefs, 2000); err != nil { + return err + } + + return nil +} + +// Function to create in batches +func createInBatches(ctx context.Context, tx *gorm.DB, brefs []map[string]any, batchSize int) error { + for i := 0; i < len(brefs); i += batchSize { + batch := brefs[i:] + if len(batch) > batchSize { + batch = batch[:batchSize] + } + + query, values := generateInsertQuery(batch) + + if err := tx.WithContext(ctx).Exec(query, values...).Error; err != nil { + return err + } + } + return nil +} + +func generateInsertQuery(brefs []map[string]any) (string, []any) { + placeholders := strings.Repeat("(?, ?, ?),", len(brefs)) + placeholders = placeholders[:len(placeholders)-1] // trim trailing comma + + query := "INSERT INTO block_refs (\"cid\", \"offset\", \"shard\") VALUES " + placeholders + + values := make([]any, 0, 3*len(brefs)) + for _, entry := range brefs { + values = append(values, entry["cid"], entry["offset"], entry["shard"]) + } + + return query, values +} diff --git a/carstore/metrics.go b/carstore/metrics.go new file mode 100644 index 000000000..0d2a0794a --- /dev/null +++ b/carstore/metrics.go @@ -0,0 +1,18 @@ +package carstore + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var writeShardFileDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "carstore_write_shard_file_duration", + Help: "Duration of writing shard file to disk", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), +}) + +var writeShardMetadataDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "carstore_write_shard_metadata_duration", + Help: "Duration of writing shard metadata to DB", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), +}) diff --git a/carstore/nonarchive.go b/carstore/nonarchive.go new file mode 100644 index 000000000..46d2e59ea --- /dev/null +++ b/carstore/nonarchive.go @@ -0,0 +1,279 @@ +package carstore + +import ( + "bytes" + "context" + "fmt" + ipld "github.com/ipfs/go-ipld-format" + "io" + "log/slog" + "sync" + + "github.com/bluesky-social/indigo/models" + blockformat "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + car "github.com/ipld/go-car" + "go.opentelemetry.io/otel" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type NonArchivalCarstore struct { + db *gorm.DB + + lk sync.Mutex + lastCommitCache map[models.Uid]*commitRefInfo + + log *slog.Logger +} + +func NewNonArchivalCarstore(db *gorm.DB) (*NonArchivalCarstore, error) { + if err := db.AutoMigrate(&commitRefInfo{}); err != nil { + return nil, err + } + + return &NonArchivalCarstore{ + db: db, + lastCommitCache: make(map[models.Uid]*commitRefInfo), + log: slog.Default().With("system", "carstorena"), + }, nil +} + +type commitRefInfo struct { + ID uint `gorm:"primarykey"` + Uid models.Uid `gorm:"uniqueIndex"` + Rev string + Root models.DbCID +} + +func (cs *NonArchivalCarstore) checkLastShardCache(user models.Uid) *commitRefInfo { + cs.lk.Lock() + defer cs.lk.Unlock() + + ls, ok := cs.lastCommitCache[user] + if ok { + return ls + } + + return nil +} + +func (cs *NonArchivalCarstore) removeLastShardCache(user models.Uid) { + cs.lk.Lock() + defer cs.lk.Unlock() + + delete(cs.lastCommitCache, user) +} + +func (cs *NonArchivalCarstore) putLastShardCache(ls *commitRefInfo) { + cs.lk.Lock() + defer cs.lk.Unlock() + + cs.lastCommitCache[ls.Uid] = ls +} + +func (cs *NonArchivalCarstore) loadCommitRefInfo(ctx context.Context, user models.Uid) (*commitRefInfo, error) { + var out commitRefInfo + wat := cs.db.Find(&out, "uid = ?", user) + if wat.Error != nil { + return nil, wat.Error + } + if wat.RowsAffected == 0 { + return nil, nil + } + return &out, nil +} + +func (cs *NonArchivalCarstore) getCommitRefInfo(ctx context.Context, user models.Uid) (*commitRefInfo, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "getCommitRefInfo") + defer span.End() + + maybeLs := cs.checkLastShardCache(user) + if maybeLs != nil { + return maybeLs, nil + } + + lastShard, err := cs.loadCommitRefInfo(ctx, user) + if err != nil { + return nil, err + } + if lastShard == nil { + return nil, nil + } + + cs.putLastShardCache(lastShard) + return lastShard, nil +} + +func (cs *NonArchivalCarstore) updateLastCommit(ctx context.Context, uid models.Uid, rev string, cid cid.Cid) error { + cri := &commitRefInfo{ + Uid: uid, + Rev: rev, + Root: models.DbCID{CID: cid}, + } + + if err := cs.db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "uid"}}, + UpdateAll: true, + }).Create(cri).Error; err != nil { + return fmt.Errorf("update or set last commit info: %w", err) + } + + cs.putLastShardCache(cri) + + return nil +} + +var commitRefZero = commitRefInfo{} + +func (cs *NonArchivalCarstore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession") + defer span.End() + + // TODO: ensure that we don't write updates on top of the wrong head + // this needs to be a compare and swap type operation + lastShard, err := cs.getCommitRefInfo(ctx, user) + if err != nil { + return nil, err + } + + if lastShard == nil { + // ok, no previous user state to refer to + lastShard = &commitRefZero + } else if since != nil && *since != lastShard.Rev { + cs.log.Warn("revision mismatch", "commitSince", since, "lastRev", lastShard.Rev, "err", ErrRepoBaseMismatch) + } + + return &DeltaSession{ + blks: make(map[cid.Cid]blockformat.Block), + base: &userView{ + user: user, + cs: cs, + prefetch: true, + cache: make(map[cid.Cid]blockformat.Block), + }, + user: user, + baseCid: lastShard.Root.CID, + cs: cs, + seq: 0, + lastRev: lastShard.Rev, + }, nil +} + +func (cs *NonArchivalCarstore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { + return &DeltaSession{ + base: &userView{ + user: user, + cs: cs, + prefetch: false, + cache: make(map[cid.Cid]blockformat.Block), + }, + readonly: true, + user: user, + cs: cs, + }, nil +} + +// TODO: incremental is only ever called true, remove the param +func (cs *NonArchivalCarstore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, w io.Writer) error { + return fmt.Errorf("not supported in non-archival mode") +} + +func (cs *NonArchivalCarstore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "ImportSlice") + defer span.End() + + carr, err := car.NewCarReader(bytes.NewReader(carslice)) + if err != nil { + return cid.Undef, nil, err + } + + if len(carr.Header.Roots) != 1 { + return cid.Undef, nil, fmt.Errorf("invalid car file, header must have a single root (has %d)", len(carr.Header.Roots)) + } + + ds, err := cs.NewDeltaSession(ctx, uid, since) + if err != nil { + return cid.Undef, nil, fmt.Errorf("new delta session failed: %w", err) + } + + var cids []cid.Cid + for { + blk, err := carr.Next() + if err != nil { + if err == io.EOF { + break + } + return cid.Undef, nil, err + } + + cids = append(cids, blk.Cid()) + + if err := ds.Put(ctx, blk); err != nil { + return cid.Undef, nil, err + } + } + + return carr.Header.Roots[0], ds, nil +} + +func (cs *NonArchivalCarstore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { + lastShard, err := cs.getCommitRefInfo(ctx, user) + if err != nil { + return cid.Undef, err + } + if lastShard == nil || lastShard.ID == 0 { + return cid.Undef, nil + } + + return lastShard.Root.CID, nil +} + +func (cs *NonArchivalCarstore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { + lastShard, err := cs.getCommitRefInfo(ctx, user) + if err != nil { + return "", err + } + if lastShard == nil || lastShard.ID == 0 { + return "", nil + } + + return lastShard.Rev, nil +} + +func (cs *NonArchivalCarstore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { + return nil, nil +} + +func (cs *NonArchivalCarstore) WipeUserData(ctx context.Context, user models.Uid) error { + if err := cs.db.Raw("DELETE from commit_ref_infos WHERE uid = ?", user).Error; err != nil { + return err + } + + cs.removeLastShardCache(user) + return nil +} + +func (cs *NonArchivalCarstore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { + return nil, fmt.Errorf("compaction not supported on non-archival") +} + +func (cs *NonArchivalCarstore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { + return nil, fmt.Errorf("compaction not supported in non-archival") +} + +func (cs *NonArchivalCarstore) HasUidCid(ctx context.Context, user models.Uid, k cid.Cid) (bool, error) { + return false, nil +} + +func (cs *NonArchivalCarstore) LookupBlockRef(ctx context.Context, k cid.Cid) (path string, offset int64, user models.Uid, err error) { + return "", 0, 0, ipld.ErrNotFound{Cid: k} +} + +func (cs *NonArchivalCarstore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { + slice, err := blocksToCar(ctx, root, rev, blks) + if err != nil { + return nil, err + } + return slice, cs.updateLastCommit(ctx, user, rev, root) +} diff --git a/carstore/repo_test.go b/carstore/repo_test.go index b679bffa5..86982ebb9 100644 --- a/carstore/repo_test.go +++ b/carstore/repo_test.go @@ -3,36 +3,46 @@ package carstore import ( "bytes" "context" + "errors" "fmt" - "io/ioutil" + "io" + "log/slog" "os" "path/filepath" "testing" "time" - "github.com/bluesky-social/indigo/api" + "github.com/bluesky-social/indigo/api/bsky" + appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/repo" - sqlbs "github.com/ipfs/go-bs-sqlite3" + "github.com/bluesky-social/indigo/util" + //sqlbs "github.com/ipfs/go-bs-sqlite3" "github.com/ipfs/go-cid" flatfs "github.com/ipfs/go-ds-flatfs" blockstore "github.com/ipfs/go-ipfs-blockstore" + ipld "github.com/ipfs/go-ipld-format" "gorm.io/driver/sqlite" "gorm.io/gorm" ) -func testCarStore() (*CarStore, func(), error) { - tempdir, err := ioutil.TempDir("", "msttest-") +func testCarStore(t testing.TB) (CarStore, func(), error) { + tempdir, err := os.MkdirTemp("", "msttest-") if err != nil { return nil, nil, err } - sharddir := filepath.Join(tempdir, "shards") - if err := os.MkdirAll(sharddir, 0775); err != nil { + sharddir1 := filepath.Join(tempdir, "shards1") + if err := os.MkdirAll(sharddir1, 0775); err != nil { + return nil, nil, err + } + + sharddir2 := filepath.Join(tempdir, "shards2") + if err := os.MkdirAll(sharddir2, 0775); err != nil { return nil, nil, err } dbstr := "file::memory:" - //dbstr := filepath.Join(tempdir, "foo.db") + //dbstr := filepath.Join(tempdir, "foo.sqlite") db, err := gorm.Open(sqlite.Open(dbstr), &gorm.Config{ SkipDefaultTransaction: true, @@ -41,7 +51,7 @@ func testCarStore() (*CarStore, func(), error) { return nil, nil, err } - cs, err := NewCarStore(db, sharddir) + cs, err := NewCarStore(db, []string{sharddir1, sharddir2}) if err != nil { return nil, nil, err } @@ -51,8 +61,25 @@ func testCarStore() (*CarStore, func(), error) { }, nil } +func testSqliteCarStore(t testing.TB) (CarStore, func(), error) { + sqs := &SQLiteStore{} + sqs.log = slogForTest(t) + err := sqs.Open(":memory:") + if err != nil { + return nil, nil, err + } + return sqs, func() {}, nil +} + +type testFactory func(t testing.TB) (CarStore, func(), error) + +var backends = map[string]testFactory{ + "cartore": testCarStore, + "sqlite": testSqliteCarStore, +} + func testFlatfsBs() (blockstore.Blockstore, func(), error) { - tempdir, err := ioutil.TempDir("", "msttest-") + tempdir, err := os.MkdirTemp("", "msttest-") if err != nil { return nil, nil, err } @@ -69,112 +96,289 @@ func testFlatfsBs() (blockstore.Blockstore, func(), error) { }, nil } -func TestBasicOperation(t *testing.T) { +func TestBasicOperation(ot *testing.T) { + ctx := context.TODO() + + for fname, tf := range backends { + ot.Run(fname, func(t *testing.T) { + + cs, cleanup, err := tf(t) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + ds, err := cs.NewDeltaSession(ctx, 1, nil) + if err != nil { + t.Fatal(err) + } + + ncid, rev, err := setupRepo(ctx, ds, false) + if err != nil { + t.Fatal(err) + } + + if _, err := ds.CloseWithRoot(ctx, ncid, rev); err != nil { + t.Fatal(err) + } + + var recs []cid.Cid + head := ncid + for i := 0; i < 10; i++ { + ds, err := cs.NewDeltaSession(ctx, 1, &rev) + if err != nil { + t.Fatal(err) + } + + rr, err := repo.OpenRepo(ctx, ds, head) + if err != nil { + t.Fatal(err) + } + + rc, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ + Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatal(err) + } + + recs = append(recs, rc) + + kmgr := &util.FakeKeyManager{} + nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) + if err != nil { + t.Fatal(err) + } + + rev = nrev + + if err := ds.CalcDiff(ctx, nil); err != nil { + t.Fatal(err) + } + + if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { + t.Fatal(err) + } + + head = nroot + } + + buf := new(bytes.Buffer) + if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { + t.Fatal(err) + } + checkRepo(t, cs, buf, recs) + + if _, err := cs.CompactUserShards(ctx, 1, false); err != nil { + t.Fatal(err) + } + + buf = new(bytes.Buffer) + if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { + t.Fatal(err) + } + checkRepo(t, cs, buf, recs) + }) + } +} + +func TestRepeatedCompactions(t *testing.T) { ctx := context.TODO() - cs, cleanup, err := testCarStore() + cs, cleanup, err := testCarStore(t) if err != nil { t.Fatal(err) } defer cleanup() - ds, err := cs.NewDeltaSession(ctx, 1, &cid.Undef) + ds, err := cs.NewDeltaSession(ctx, 1, nil) if err != nil { t.Fatal(err) } - ncid, err := setupRepo(ctx, ds) + ncid, rev, err := setupRepo(ctx, ds, false) if err != nil { t.Fatal(err) } - if _, err := ds.CloseWithRoot(ctx, ncid); err != nil { + if _, err := ds.CloseWithRoot(ctx, ncid, rev); err != nil { t.Fatal(err) } + var recs []cid.Cid head := ncid - for i := 0; i < 10; i++ { - ds, err := cs.NewDeltaSession(ctx, 1, &head) - if err != nil { - t.Fatal(err) - } - rr, err := repo.OpenRepo(ctx, ds, head) + var lastRec string + + for loop := 0; loop < 50; loop++ { + for i := 0; i < 20; i++ { + ds, err := cs.NewDeltaSession(ctx, 1, &rev) + if err != nil { + t.Fatal(err) + } + + rr, err := repo.OpenRepo(ctx, ds, head) + if err != nil { + t.Fatal(err) + } + if i%4 == 3 { + if err := rr.DeleteRecord(ctx, lastRec); err != nil { + t.Fatal(err) + } + recs = recs[:len(recs)-1] + } else { + rc, tid, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ + Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatal(err) + } + + recs = append(recs, rc) + lastRec = "app.bsky.feed.post/" + tid + } + + kmgr := &util.FakeKeyManager{} + nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) + if err != nil { + t.Fatal(err) + } + + rev = nrev + + if err := ds.CalcDiff(ctx, nil); err != nil { + t.Fatal(err) + } + + if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { + t.Fatal(err) + } + + head = nroot + } + fmt.Println("Run compaction", loop) + st, err := cs.CompactUserShards(ctx, 1, false) if err != nil { t.Fatal(err) } - if _, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &api.PostRecord{ - Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), - }); err != nil { + fmt.Printf("%#v\n", st) + + buf := new(bytes.Buffer) + if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { t.Fatal(err) } + checkRepo(t, cs, buf, recs) + } - nroot, err := rr.Commit(ctx) - if err != nil { - t.Fatal(err) + buf := new(bytes.Buffer) + if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { + t.Fatal(err) + } + checkRepo(t, cs, buf, recs) +} + +func checkRepo(t *testing.T, cs CarStore, r io.Reader, expRecs []cid.Cid) { + t.Helper() + rep, err := repo.ReadRepoFromCar(context.TODO(), r) + if err != nil { + t.Fatal("Reading repo: ", err) + } + + set := make(map[cid.Cid]bool) + for _, c := range expRecs { + set[c] = true + } + + if err := rep.ForEach(context.TODO(), "", func(k string, v cid.Cid) error { + if !set[v] { + return fmt.Errorf("have record we did not expect") } - if _, err := ds.CloseWithRoot(ctx, nroot); err != nil { - t.Fatal(err) + delete(set, v) + return nil + + }); err != nil { + var ierr ipld.ErrNotFound + if errors.As(err, &ierr) { + fmt.Println("matched error") + bs, err := cs.ReadOnlySession(1) + if err != nil { + fmt.Println("could not read session: ", err) + } + + blk, err := bs.Get(context.TODO(), ierr.Cid) + if err != nil { + fmt.Println("also failed the local get: ", err) + } else { + fmt.Println("LOCAL GET SUCCESS", len(blk.RawData())) + } } - head = nroot + t.Fatal("walking repo: ", err) } - buf := new(bytes.Buffer) - if err := cs.ReadUserCar(ctx, 1, cid.Undef, true, buf); err != nil { - t.Fatal(err) + if len(set) > 0 { + t.Fatalf("expected to find more cids in repo: %v", set) } - fmt.Println(buf.Len()) - } -func setupRepo(ctx context.Context, bs blockstore.Blockstore) (cid.Cid, error) { - nr := repo.NewRepo(ctx, bs) +func setupRepo(ctx context.Context, bs blockstore.Blockstore, mkprofile bool) (cid.Cid, string, error) { + nr := repo.NewRepo(ctx, "did:foo", bs) - if _, _, err := nr.CreateRecord(ctx, "app.bsky.feed.post", &api.PostRecord{ - Text: fmt.Sprintf("hey look its a tweet %s", time.Now()), - }); err != nil { - return cid.Undef, err + if mkprofile { + _, err := nr.PutRecord(ctx, "app.bsky.actor.profile/self", &bsky.ActorProfile{}) + if err != nil { + return cid.Undef, "", fmt.Errorf("write record failed: %w", err) + } } - ncid, err := nr.Commit(ctx) + kmgr := &util.FakeKeyManager{} + ncid, rev, err := nr.Commit(ctx, kmgr.SignForUser) if err != nil { - return cid.Undef, err + return cid.Undef, "", fmt.Errorf("commit failed: %w", err) } - return ncid, nil + return ncid, rev, nil } func BenchmarkRepoWritesCarstore(b *testing.B) { ctx := context.TODO() - cs, cleanup, err := testCarStore() + cs, cleanup, err := testCarStore(b) + innerBenchmarkRepoWritesCarstore(b, ctx, cs, cleanup, err) +} +func BenchmarkRepoWritesSqliteCarstore(b *testing.B) { + ctx := context.TODO() + + cs, cleanup, err := testSqliteCarStore(b) + innerBenchmarkRepoWritesCarstore(b, ctx, cs, cleanup, err) +} +func innerBenchmarkRepoWritesCarstore(b *testing.B, ctx context.Context, cs CarStore, cleanup func(), err error) { if err != nil { b.Fatal(err) } defer cleanup() - ds, err := cs.NewDeltaSession(ctx, 1, &cid.Undef) + ds, err := cs.NewDeltaSession(ctx, 1, nil) if err != nil { b.Fatal(err) } - ncid, err := setupRepo(ctx, ds) + ncid, rev, err := setupRepo(ctx, ds, false) if err != nil { b.Fatal(err) } - if _, err := ds.CloseWithRoot(ctx, ncid); err != nil { + if _, err := ds.CloseWithRoot(ctx, ncid, rev); err != nil { b.Fatal(err) } head := ncid b.ResetTimer() for i := 0; i < b.N; i++ { - ds, err := cs.NewDeltaSession(ctx, 1, &head) + ds, err := cs.NewDeltaSession(ctx, 1, &rev) if err != nil { b.Fatal(err) } @@ -184,18 +388,24 @@ func BenchmarkRepoWritesCarstore(b *testing.B) { b.Fatal(err) } - if _, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &api.PostRecord{ + if _, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ Text: fmt.Sprintf("hey look its a tweet %s", time.Now()), }); err != nil { b.Fatal(err) } - nroot, err := rr.Commit(ctx) + kmgr := &util.FakeKeyManager{} + nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) if err != nil { b.Fatal(err) } - if _, err := ds.CloseWithRoot(ctx, nroot); err != nil { + rev = nrev + if err := ds.CalcDiff(ctx, nil); err != nil { + b.Fatal(err) + } + + if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { b.Fatal(err) } @@ -212,7 +422,7 @@ func BenchmarkRepoWritesFlatfs(b *testing.B) { } defer cleanup() - ncid, err := setupRepo(ctx, bs) + ncid, _, err := setupRepo(ctx, bs, false) if err != nil { b.Fatal(err) } @@ -226,13 +436,14 @@ func BenchmarkRepoWritesFlatfs(b *testing.B) { b.Fatal(err) } - if _, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &api.PostRecord{ + if _, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ Text: fmt.Sprintf("hey look its a tweet %s", time.Now()), }); err != nil { b.Fatal(err) } - nroot, err := rr.Commit(ctx) + kmgr := &util.FakeKeyManager{} + nroot, _, err := rr.Commit(ctx, kmgr.SignForUser) if err != nil { b.Fatal(err) } @@ -241,6 +452,7 @@ func BenchmarkRepoWritesFlatfs(b *testing.B) { } } +/* NOTE(bnewbold): this depends on github.com/ipfs/go-bs-sqlite3, which rewrote git history (?) breaking the dependency tree. We can roll forward, but that will require broad dependency updates. So for now just removing this benchmark/perf test. func BenchmarkRepoWritesSqlite(b *testing.B) { ctx := context.TODO() @@ -249,7 +461,7 @@ func BenchmarkRepoWritesSqlite(b *testing.B) { b.Fatal(err) } - ncid, err := setupRepo(ctx, bs) + ncid, _, err := setupRepo(ctx, bs, false) if err != nil { b.Fatal(err) } @@ -263,13 +475,14 @@ func BenchmarkRepoWritesSqlite(b *testing.B) { b.Fatal(err) } - if _, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &api.PostRecord{ + if _, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ Text: fmt.Sprintf("hey look its a tweet %s", time.Now()), }); err != nil { b.Fatal(err) } - nroot, err := rr.Commit(ctx) + kmgr := &util.FakeKeyManager{} + nroot, _, err := rr.Commit(ctx, kmgr.SignForUser) if err != nil { b.Fatal(err) } @@ -277,3 +490,154 @@ func BenchmarkRepoWritesSqlite(b *testing.B) { head = nroot } } +*/ + +func TestDuplicateBlockAcrossShards(ot *testing.T) { + ctx := context.TODO() + + for fname, tf := range backends { + ot.Run(fname, func(t *testing.T) { + + cs, cleanup, err := tf(t) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + ds1, err := cs.NewDeltaSession(ctx, 1, nil) + if err != nil { + t.Fatal(err) + } + + ds2, err := cs.NewDeltaSession(ctx, 2, nil) + if err != nil { + t.Fatal(err) + } + + ds3, err := cs.NewDeltaSession(ctx, 3, nil) + if err != nil { + t.Fatal(err) + } + + var cids []cid.Cid + var revs []string + for _, ds := range []*DeltaSession{ds1, ds2, ds3} { + ncid, rev, err := setupRepo(ctx, ds, true) + if err != nil { + t.Fatal(err) + } + + if _, err := ds.CloseWithRoot(ctx, ncid, rev); err != nil { + t.Fatal(err) + } + cids = append(cids, ncid) + revs = append(revs, rev) + } + + var recs []cid.Cid + head := cids[1] + rev := revs[1] + for i := 0; i < 10; i++ { + ds, err := cs.NewDeltaSession(ctx, 2, &rev) + if err != nil { + t.Fatal(err) + } + + rr, err := repo.OpenRepo(ctx, ds, head) + if err != nil { + t.Fatal(err) + } + + rc, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ + Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatal(err) + } + + recs = append(recs, rc) + + kmgr := &util.FakeKeyManager{} + nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) + if err != nil { + t.Fatal(err) + } + + rev = nrev + + if err := ds.CalcDiff(ctx, nil); err != nil { + t.Fatal(err) + } + + if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { + t.Fatal(err) + } + + head = nroot + } + + // explicitly update the profile object + { + ds, err := cs.NewDeltaSession(ctx, 2, &rev) + if err != nil { + t.Fatal(err) + } + + rr, err := repo.OpenRepo(ctx, ds, head) + if err != nil { + t.Fatal(err) + } + + desc := "this is so unique" + rc, err := rr.UpdateRecord(ctx, "app.bsky.actor.profile/self", &appbsky.ActorProfile{ + Description: &desc, + }) + if err != nil { + t.Fatal(err) + } + + recs = append(recs, rc) + + kmgr := &util.FakeKeyManager{} + nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) + if err != nil { + t.Fatal(err) + } + + rev = nrev + + if err := ds.CalcDiff(ctx, nil); err != nil { + t.Fatal(err) + } + + if _, err := ds.CloseWithRoot(ctx, nroot, rev); err != nil { + t.Fatal(err) + } + + head = nroot + } + + buf := new(bytes.Buffer) + if err := cs.ReadUserCar(ctx, 2, "", true, buf); err != nil { + t.Fatal(err) + } + checkRepo(t, cs, buf, recs) + }) + } +} + +type testWriter struct { + t testing.TB +} + +func (tw testWriter) Write(p []byte) (n int, err error) { + tw.t.Log(string(p)) + return len(p), nil +} + +func slogForTest(t testing.TB) *slog.Logger { + hopts := slog.HandlerOptions{ + Level: slog.LevelDebug, + } + return slog.New(slog.NewTextHandler(&testWriter{t}, &hopts)) +} diff --git a/carstore/scylla.go b/carstore/scylla.go new file mode 100644 index 000000000..e78eadac4 --- /dev/null +++ b/carstore/scylla.go @@ -0,0 +1,638 @@ +//go:build scylla + +package carstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/bluesky-social/indigo/models" + "github.com/gocql/gocql" + blockformat "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipld/go-car" + _ "github.com/mattn/go-sqlite3" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "io" + "log/slog" + "math" + "math/rand/v2" + "time" +) + +type ScyllaStore struct { + WriteSession *gocql.Session + ReadSession *gocql.Session + + // scylla servers + scyllaAddrs []string + // scylla namespace where we find our table + keyspace string + + log *slog.Logger + + lastShardCache lastShardCache +} + +func NewScyllaStore(addrs []string, keyspace string) (*ScyllaStore, error) { + out := new(ScyllaStore) + out.scyllaAddrs = addrs + out.keyspace = keyspace + err := out.Open() + if err != nil { + return nil, err + } + return out, nil +} + +func (sqs *ScyllaStore) Open() error { + if sqs.log == nil { + sqs.log = slog.Default() + } + sqs.log.Debug("scylla connect", "addrs", sqs.scyllaAddrs) + var err error + + // + // Write session + // + var writeSession *gocql.Session + for retry := 0; ; retry++ { + writeCluster := gocql.NewCluster(sqs.scyllaAddrs...) + writeCluster.Keyspace = sqs.keyspace + // Default port, the client should automatically upgrade to shard-aware port + writeCluster.Port = 9042 + writeCluster.Consistency = gocql.Quorum + writeCluster.RetryPolicy = &ExponentialBackoffRetryPolicy{NumRetries: 10, Min: 100 * time.Millisecond, Max: 10 * time.Second} + writeCluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(gocql.RoundRobinHostPolicy()) + writeSession, err = writeCluster.CreateSession() + if err != nil { + if retry > 200 { + return fmt.Errorf("failed to connect read session too many times: %w", err) + } + sqs.log.Error("failed to connect to ScyllaDB Read Session, retrying in 1s", "retry", retry, "err", err) + time.Sleep(delayForAttempt(retry)) + continue + } + break + } + + // + // Read session + // + var readSession *gocql.Session + for retry := 0; ; retry++ { + readCluster := gocql.NewCluster(sqs.scyllaAddrs...) + readCluster.Keyspace = sqs.keyspace + // Default port, the client should automatically upgrade to shard-aware port + readCluster.Port = 9042 + readCluster.RetryPolicy = &ExponentialBackoffRetryPolicy{NumRetries: 5, Min: 10 * time.Millisecond, Max: 1 * time.Second} + readCluster.Consistency = gocql.One + readCluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(gocql.RoundRobinHostPolicy()) + readSession, err = readCluster.CreateSession() + if err != nil { + if retry > 200 { + return fmt.Errorf("failed to connect read session too many times: %w", err) + } + sqs.log.Error("failed to connect to ScyllaDB Read Session, retrying in 1s", "retry", retry, "err", err) + time.Sleep(delayForAttempt(retry)) + continue + } + break + } + + sqs.WriteSession = writeSession + sqs.ReadSession = readSession + + err = sqs.createTables() + if err != nil { + return fmt.Errorf("scylla could not create tables, %w", err) + } + sqs.lastShardCache.source = sqs + sqs.lastShardCache.Init() + return nil +} + +var createTableTexts = []string{ + `CREATE TABLE IF NOT EXISTS blocks (uid bigint, cid blob, rev varchar, root blob, block blob, PRIMARY KEY((uid,cid)))`, + // This is the INDEX I wish we could use, but scylla can't do it so we MATERIALIZED VIEW instead + //`CREATE INDEX IF NOT EXISTS block_by_rev ON blocks (uid, rev)`, + `CREATE MATERIALIZED VIEW IF NOT EXISTS blocks_by_uidrev +AS SELECT uid, rev, cid, root +FROM blocks +WHERE uid IS NOT NULL AND rev IS NOT NULL AND cid IS NOT NULL +PRIMARY KEY ((uid), rev, cid) WITH CLUSTERING ORDER BY (rev DESC)`, +} + +func (sqs *ScyllaStore) createTables() error { + for i, text := range createTableTexts { + err := sqs.WriteSession.Query(text).Exec() + if err != nil { + return fmt.Errorf("scylla create table statement [%d] %v: %w", i, text, err) + } + } + return nil +} + +// writeNewShard needed for DeltaSession.CloseWithRoot +func (sqs *ScyllaStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { + scWriteNewShard.Inc() + sqs.log.Debug("write shard", "uid", user, "root", root, "rev", rev, "nblocks", len(blks)) + start := time.Now() + ctx, span := otel.Tracer("carstore").Start(ctx, "writeNewShard") + defer span.End() + buf := new(bytes.Buffer) + hnw, err := WriteCarHeader(buf, root) + if err != nil { + return nil, fmt.Errorf("failed to write car header: %w", err) + } + offset := hnw + + dbroot := root.Bytes() + + span.SetAttributes(attribute.Int("blocks", len(blks))) + + for bcid, block := range blks { + // build shard for output firehose + nw, err := LdWrite(buf, bcid.Bytes(), block.RawData()) + if err != nil { + return nil, fmt.Errorf("failed to write block: %w", err) + } + offset += nw + + // TODO: scylla BATCH doesn't apply if the batch crosses partition keys; BUT, we may be able to send many blocks concurrently? + dbcid := bcid.Bytes() + blockbytes := block.RawData() + // we're relying on cql auto-prepare, no 'PreparedStatement' + err = sqs.WriteSession.Query( + `INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?)`, + user, dbcid, rev, dbroot, blockbytes, + ).Idempotent(true).Exec() + if err != nil { + return nil, fmt.Errorf("(uid,cid) block store failed, %w", err) + } + sqs.log.Debug("put block", "uid", user, "cid", bcid, "size", len(blockbytes)) + } + + shard := CarShard{ + Root: models.DbCID{CID: root}, + DataStart: hnw, + Seq: seq, + Usr: user, + Rev: rev, + } + + sqs.lastShardCache.put(&shard) + + dt := time.Since(start).Seconds() + scWriteTimes.Observe(dt) + return buf.Bytes(), nil +} + +// GetLastShard nedeed for NewDeltaSession indirectly through lastShardCache +// What we actually seem to need from this: last {Rev, Root.CID} +func (sqs *ScyllaStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarShard, error) { + scGetLastShard.Inc() + var rev string + var rootb []byte + err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks_by_uidrev WHERE uid = ? ORDER BY rev DESC LIMIT 1`, uid).Scan(&rev, &rootb) + if errors.Is(err, gocql.ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("last shard err, %w", err) + } + xcid, cidErr := cid.Cast(rootb) + if cidErr != nil { + return nil, fmt.Errorf("last shard bad cid, %w", cidErr) + } + return &CarShard{ + Root: models.DbCID{CID: xcid}, + Rev: rev, + }, nil +} + +func (sqs *ScyllaStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { + sqs.log.Warn("TODO: don't call compaction") + return nil, nil +} + +func (sqs *ScyllaStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { + sqs.log.Warn("TODO: don't call compaction targets") + return nil, nil +} + +func (sqs *ScyllaStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { + // TODO: same as FileCarStore; re-unify + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return cid.Undef, err + } + if lastShard == nil { + return cid.Undef, nil + } + if lastShard.ID == 0 { + return cid.Undef, nil + } + + return lastShard.Root.CID, nil +} + +func (sqs *ScyllaStore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { + // TODO: same as FileCarStore; re-unify + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return "", err + } + if lastShard == nil { + return "", nil + } + if lastShard.ID == 0 { + return "", nil + } + + return lastShard.Rev, nil +} + +func (sqs *ScyllaStore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { + // TODO: same as FileCarStore, re-unify + ctx, span := otel.Tracer("carstore").Start(ctx, "ImportSlice") + defer span.End() + + carr, err := car.NewCarReader(bytes.NewReader(carslice)) + if err != nil { + return cid.Undef, nil, err + } + + if len(carr.Header.Roots) != 1 { + return cid.Undef, nil, fmt.Errorf("invalid car file, header must have a single root (has %d)", len(carr.Header.Roots)) + } + + ds, err := sqs.NewDeltaSession(ctx, uid, since) + if err != nil { + return cid.Undef, nil, fmt.Errorf("new delta session failed: %w", err) + } + + var cids []cid.Cid + for { + blk, err := carr.Next() + if err != nil { + if err == io.EOF { + break + } + return cid.Undef, nil, err + } + + cids = append(cids, blk.Cid()) + + if err := ds.Put(ctx, blk); err != nil { + return cid.Undef, nil, err + } + } + + return carr.Header.Roots[0], ds, nil +} + +func (sqs *ScyllaStore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession") + defer span.End() + + // TODO: ensure that we don't write updates on top of the wrong head + // this needs to be a compare and swap type operation + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return nil, fmt.Errorf("NewDeltaSession, lsc, %w", err) + } + + if lastShard == nil { + lastShard = &zeroShard + } + + if since != nil && *since != lastShard.Rev { + return nil, fmt.Errorf("revision mismatch: %s != %s: %w", *since, lastShard.Rev, ErrRepoBaseMismatch) + } + + return &DeltaSession{ + blks: make(map[cid.Cid]blockformat.Block), + base: &sqliteUserView{ + uid: user, + sqs: sqs, + }, + user: user, + baseCid: lastShard.Root.CID, + cs: sqs, + seq: lastShard.Seq + 1, + lastRev: lastShard.Rev, + }, nil +} + +func (sqs *ScyllaStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { + return &DeltaSession{ + base: &sqliteUserView{ + uid: user, + sqs: sqs, + }, + readonly: true, + user: user, + cs: sqs, + }, nil +} + +// ReadUserCar +// incremental is only ever called true +func (sqs *ScyllaStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, shardOut io.Writer) error { + scGetCar.Inc() + ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") + defer span.End() + start := time.Now() + + cidchan := make(chan cid.Cid, 100) + + go func() { + defer close(cidchan) + cids := sqs.ReadSession.Query(`SELECT cid FROM blocks_by_uidrev WHERE uid = ? AND rev > ? ORDER BY rev DESC`, user, sinceRev).Iter() + defer cids.Close() + for { + var cidb []byte + ok := cids.Scan(&cidb) + if !ok { + break + } + xcid, cidErr := cid.Cast(cidb) + if cidErr != nil { + sqs.log.Warn("ReadUserCar bad cid", "err", cidErr) + continue + } + cidchan <- xcid + } + }() + nblocks := 0 + first := true + for xcid := range cidchan { + var xrev string + var xroot []byte + var xblock []byte + err := sqs.ReadSession.Query("SELECT rev, root, block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, xcid.Bytes()).Scan(&xrev, &xroot, &xblock) + if err != nil { + return fmt.Errorf("rcar bad read, %w", err) + } + if first { + rootCid, cidErr := cid.Cast(xroot) + if cidErr != nil { + return fmt.Errorf("rcar bad rootcid, %w", err) + } + if err := car.WriteHeader(&car.CarHeader{ + Roots: []cid.Cid{rootCid}, + Version: 1, + }, shardOut); err != nil { + return fmt.Errorf("rcar bad header, %w", err) + } + first = false + } + nblocks++ + _, err = LdWrite(shardOut, xcid.Bytes(), xblock) + if err != nil { + return fmt.Errorf("rcar bad write, %w", err) + } + } + span.SetAttributes(attribute.Int("blocks", nblocks)) + sqs.log.Debug("read car", "nblocks", nblocks, "since", sinceRev) + scReadCarTimes.Observe(time.Since(start).Seconds()) + return nil +} + +// Stat is only used in a debugging admin handler +// don't bother implementing it (for now?) +func (sqs *ScyllaStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { + sqs.log.Warn("Stat debugging method not implemented for sqlite store") + return nil, nil +} + +func (sqs *ScyllaStore) WipeUserData(ctx context.Context, user models.Uid) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "WipeUserData") + defer span.End() + + // LOL, can't do this if primary key is (uid,cid) because that's hashed with no scan! + //err := sqs.WriteSession.Query("DELETE FROM blocks WHERE uid = ?", user).Exec() + + cidchan := make(chan cid.Cid, 100) + + go func() { + defer close(cidchan) + cids := sqs.ReadSession.Query(`SELECT cid FROM blocks_by_uidrev WHERE uid = ?`, user).Iter() + defer cids.Close() + for { + var cidb []byte + ok := cids.Scan(&cidb) + if !ok { + break + } + xcid, cidErr := cid.Cast(cidb) + if cidErr != nil { + sqs.log.Warn("ReadUserCar bad cid", "err", cidErr) + continue + } + cidchan <- xcid + } + }() + nblocks := 0 + errcount := 0 + for xcid := range cidchan { + err := sqs.ReadSession.Query("DELETE FROM blocks WHERE uid = ? AND cid = ?", user, xcid.Bytes()).Exec() + if err != nil { + sqs.log.Warn("ReadUserCar bad delete", "err", err) + errcount++ + if errcount > 10 { + return fmt.Errorf("ReadUserCar bad delete: %w", err) + } + } + nblocks++ + } + scUsersWiped.Inc() + scBlocksDeleted.Add(float64(nblocks)) + return nil +} + +// HasUidCid needed for NewDeltaSession userView +func (sqs *ScyllaStore) HasUidCid(ctx context.Context, user models.Uid, bcid cid.Cid) (bool, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + scHas.Inc() + var rev string + var rootb []byte + err := sqs.ReadSession.Query(`SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1`, user, bcid.Bytes()).Scan(&rev, &rootb) + if err != nil { + return false, fmt.Errorf("hasUC bad scan, %w", err) + } + return true, nil +} + +func (sqs *ScyllaStore) CarStore() CarStore { + return sqs +} + +func (sqs *ScyllaStore) Close() error { + sqs.WriteSession.Close() + sqs.ReadSession.Close() + return nil +} + +func (sqs *ScyllaStore) getBlock(ctx context.Context, user models.Uid, bcid cid.Cid) (blockformat.Block, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + scGetBlock.Inc() + start := time.Now() + var blockb []byte + err := sqs.ReadSession.Query("SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, bcid.Bytes()).Scan(&blockb) + if err != nil { + return nil, fmt.Errorf("getb err, %w", err) + } + dt := time.Since(start) + scGetTimes.Observe(dt.Seconds()) + return blocks.NewBlock(blockb), nil +} + +func (sqs *ScyllaStore) getBlockSize(ctx context.Context, user models.Uid, bcid cid.Cid) (int64, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + scGetBlockSize.Inc() + var out int64 + err := sqs.ReadSession.Query("SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1", user, bcid.Bytes()).Scan(&out) + if err != nil { + return 0, fmt.Errorf("getbs err, %w", err) + } + return out, nil +} + +var scUsersWiped = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_users_wiped", + Help: "User rows deleted in scylla backend", +}) + +var scBlocksDeleted = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_blocks_deleted", + Help: "User blocks deleted in scylla backend", +}) + +var scGetBlock = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_get_block", + Help: "get block scylla backend", +}) + +var scGetBlockSize = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_get_block_size", + Help: "get block size scylla backend", +}) + +var scGetCar = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_get_car", + Help: "get block scylla backend", +}) + +var scHas = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_has", + Help: "check block presence scylla backend", +}) + +var scGetLastShard = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_get_last_shard", + Help: "get last shard scylla backend", +}) + +var scWriteNewShard = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sc_write_shard", + Help: "write shard blocks scylla backend", +}) + +var timeBuckets []float64 +var scWriteTimes prometheus.Histogram +var scGetTimes prometheus.Histogram +var scReadCarTimes prometheus.Histogram + +func init() { + timeBuckets = make([]float64, 1, 20) + timeBuckets[0] = 0.000_0100 + i := 0 + for timeBuckets[i] < 1 && len(timeBuckets) < 20 { + timeBuckets = append(timeBuckets, timeBuckets[i]*2) + i++ + } + scWriteTimes = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "bgs_sc_write_times", + Buckets: timeBuckets, + }) + scGetTimes = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "bgs_sc_get_times", + Buckets: timeBuckets, + }) + scReadCarTimes = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "bgs_sc_readcar_times", + Buckets: timeBuckets, + }) +} + +// TODO: copied from tango, re-unify? +// ExponentialBackoffRetryPolicy sleeps between attempts +type ExponentialBackoffRetryPolicy struct { + NumRetries int + Min, Max time.Duration +} + +func (e *ExponentialBackoffRetryPolicy) napTime(attempts int) time.Duration { + return getExponentialTime(e.Min, e.Max, attempts) +} + +func (e *ExponentialBackoffRetryPolicy) Attempt(q gocql.RetryableQuery) bool { + if q.Attempts() > e.NumRetries { + return false + } + time.Sleep(e.napTime(q.Attempts())) + return true +} + +// used to calculate exponentially growing time +func getExponentialTime(min time.Duration, max time.Duration, attempts int) time.Duration { + if min <= 0 { + min = 100 * time.Millisecond + } + if max <= 0 { + max = 10 * time.Second + } + minFloat := float64(min) + napDuration := minFloat * math.Pow(2, float64(attempts-1)) + // add some jitter + napDuration += rand.Float64()*minFloat - (minFloat / 2) + if napDuration > float64(max) { + return time.Duration(max) + } + return time.Duration(napDuration) +} + +// GetRetryType returns the retry type for the given error +func (e *ExponentialBackoffRetryPolicy) GetRetryType(err error) gocql.RetryType { + // Retry timeouts and/or contention errors on the same host + if errors.Is(err, gocql.ErrTimeoutNoResponse) || + errors.Is(err, gocql.ErrNoStreams) || + errors.Is(err, gocql.ErrTooManyTimeouts) { + return gocql.Retry + } + + // Retry next host on unavailable errors + if errors.Is(err, gocql.ErrUnavailable) || + errors.Is(err, gocql.ErrConnectionClosed) || + errors.Is(err, gocql.ErrSessionClosed) { + return gocql.RetryNextHost + } + + // Otherwise don't retry + return gocql.Rethrow +} + +func delayForAttempt(attempt int) time.Duration { + if attempt < 50 { + return time.Millisecond * 5 + } + + return time.Second +} diff --git a/carstore/scylla_stub.go b/carstore/scylla_stub.go new file mode 100644 index 000000000..863bca489 --- /dev/null +++ b/carstore/scylla_stub.go @@ -0,0 +1,9 @@ +//go:build !scylla + +package carstore + +import "errors" + +func NewScyllaStore(addrs []string, keyspace string) (CarStore, error) { + return nil, errors.New("scylla compiled out") +} diff --git a/carstore/sqlite_store.go b/carstore/sqlite_store.go new file mode 100644 index 000000000..5fbf4badb --- /dev/null +++ b/carstore/sqlite_store.go @@ -0,0 +1,575 @@ +package carstore + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + + "go.opentelemetry.io/otel/attribute" + + "github.com/bluesky-social/indigo/models" + blockformat "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipld/go-car" + _ "github.com/mattn/go-sqlite3" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "go.opentelemetry.io/otel" +) + +// var log = logging.Logger("sqstore") + +type SQLiteStore struct { + dbPath string + db *sql.DB + + log *slog.Logger + + lastShardCache lastShardCache +} + +func ensureDir(path string) error { + fi, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return os.MkdirAll(path, 0755) + } + return err + } + if fi.IsDir() { + return nil + } + return fmt.Errorf("%s exists but is not a directory", path) +} + +func NewSqliteStore(csdir string) (*SQLiteStore, error) { + if err := ensureDir(csdir); err != nil { + return nil, err + } + dbpath := filepath.Join(csdir, "db.sqlite3") + out := new(SQLiteStore) + err := out.Open(dbpath) + if err != nil { + return nil, err + } + return out, nil +} + +func (sqs *SQLiteStore) Open(path string) error { + if sqs.log == nil { + sqs.log = slog.Default() + } + sqs.log.Debug("open db", "path", path) + db, err := sql.Open("sqlite3", path) + if err != nil { + return fmt.Errorf("%s: sqlite could not open, %w", path, err) + } + sqs.db = db + sqs.dbPath = path + err = sqs.createTables() + if err != nil { + return fmt.Errorf("%s: sqlite could not create tables, %w", path, err) + } + sqs.lastShardCache.source = sqs + sqs.lastShardCache.Init() + return nil +} + +func (sqs *SQLiteStore) createTables() error { + tx, err := sqs.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + _, err = tx.Exec("CREATE TABLE IF NOT EXISTS blocks (uid int, cid blob, rev varchar, root blob, block blob, PRIMARY KEY(uid,cid));") + if err != nil { + return fmt.Errorf("%s: create table blocks..., %w", sqs.dbPath, err) + } + _, err = tx.Exec("CREATE INDEX IF NOT EXISTS blocx_by_rev ON blocks (uid, rev DESC)") + if err != nil { + return fmt.Errorf("%s: create blocks by rev index, %w", sqs.dbPath, err) + } + return tx.Commit() +} + +// writeNewShard needed for DeltaSession.CloseWithRoot +func (sqs *SQLiteStore) writeNewShard(ctx context.Context, root cid.Cid, rev string, user models.Uid, seq int, blks map[cid.Cid]blockformat.Block, rmcids map[cid.Cid]bool) ([]byte, error) { + sqWriteNewShard.Inc() + sqs.log.Debug("write shard", "uid", user, "root", root, "rev", rev, "nblocks", len(blks)) + ctx, span := otel.Tracer("carstore").Start(ctx, "writeNewShard") + defer span.End() + // this is "write many blocks", "write one block" is above in putBlock(). keep them in sync. + buf := new(bytes.Buffer) + hnw, err := WriteCarHeader(buf, root) + if err != nil { + return nil, fmt.Errorf("failed to write car header: %w", err) + } + offset := hnw + + tx, err := sqs.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("bad block insert tx, %w", err) + } + defer tx.Rollback() + insertStatement, err := tx.PrepareContext(ctx, "INSERT INTO blocks (uid, cid, rev, root, block) VALUES (?, ?, ?, ?, ?) ON CONFLICT (uid,cid) DO UPDATE SET rev=excluded.rev, root=excluded.root, block=excluded.block") + if err != nil { + return nil, fmt.Errorf("bad block insert sql, %w", err) + } + defer insertStatement.Close() + + dbroot := models.DbCID{CID: root} + + span.SetAttributes(attribute.Int("blocks", len(blks))) + + for bcid, block := range blks { + // build shard for output firehose + nw, err := LdWrite(buf, bcid.Bytes(), block.RawData()) + if err != nil { + return nil, fmt.Errorf("failed to write block: %w", err) + } + offset += nw + + // TODO: better databases have an insert-many option for a prepared statement + dbcid := models.DbCID{CID: bcid} + blockbytes := block.RawData() + _, err = insertStatement.ExecContext(ctx, user, dbcid, rev, dbroot, blockbytes) + if err != nil { + return nil, fmt.Errorf("(uid,cid) block store failed, %w", err) + } + sqs.log.Debug("put block", "uid", user, "cid", bcid, "size", len(blockbytes)) + } + err = tx.Commit() + if err != nil { + return nil, fmt.Errorf("bad block insert commit, %w", err) + } + + shard := CarShard{ + Root: models.DbCID{CID: root}, + DataStart: hnw, + Seq: seq, + Usr: user, + Rev: rev, + } + + sqs.lastShardCache.put(&shard) + + return buf.Bytes(), nil +} + +var ErrNothingThere = errors.New("nothing to read)") + +// GetLastShard nedeed for NewDeltaSession indirectly through lastShardCache +// What we actually seem to need from this: last {Rev, Root.CID} +func (sqs *SQLiteStore) GetLastShard(ctx context.Context, uid models.Uid) (*CarShard, error) { + sqGetLastShard.Inc() + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return nil, fmt.Errorf("bad last shard tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT rev, root FROM blocks WHERE uid = ? ORDER BY rev DESC LIMIT 1") + if err != nil { + return nil, fmt.Errorf("bad last shard sql, %w", err) + } + rows, err := qstmt.QueryContext(ctx, uid) + if err != nil { + return nil, fmt.Errorf("last shard err, %w", err) + } + if rows.Next() { + var rev string + var rootb models.DbCID + err = rows.Scan(&rev, &rootb) + if err != nil { + return nil, fmt.Errorf("last shard bad scan, %w", err) + } + return &CarShard{ + Root: rootb, + Rev: rev, + }, nil + } + return nil, nil +} + +func (sqs *SQLiteStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { + sqs.log.Warn("TODO: don't call compaction") + return nil, nil +} + +func (sqs *SQLiteStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { + sqs.log.Warn("TODO: don't call compaction targets") + return nil, nil +} + +func (sqs *SQLiteStore) GetUserRepoHead(ctx context.Context, user models.Uid) (cid.Cid, error) { + // TODO: same as FileCarStore; re-unify + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return cid.Undef, err + } + if lastShard == nil { + return cid.Undef, nil + } + + return lastShard.Root.CID, nil +} + +func (sqs *SQLiteStore) GetUserRepoRev(ctx context.Context, user models.Uid) (string, error) { + // TODO: same as FileCarStore; re-unify + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return "", err + } + if lastShard == nil { + return "", nil + } + + return lastShard.Rev, nil +} + +func (sqs *SQLiteStore) ImportSlice(ctx context.Context, uid models.Uid, since *string, carslice []byte) (cid.Cid, *DeltaSession, error) { + // TODO: same as FileCarStore, re-unify + ctx, span := otel.Tracer("carstore").Start(ctx, "ImportSlice") + defer span.End() + + carr, err := car.NewCarReader(bytes.NewReader(carslice)) + if err != nil { + return cid.Undef, nil, err + } + + if len(carr.Header.Roots) != 1 { + return cid.Undef, nil, fmt.Errorf("invalid car file, header must have a single root (has %d)", len(carr.Header.Roots)) + } + + ds, err := sqs.NewDeltaSession(ctx, uid, since) + if err != nil { + return cid.Undef, nil, fmt.Errorf("new delta session failed: %w", err) + } + + var cids []cid.Cid + for { + blk, err := carr.Next() + if err != nil { + if err == io.EOF { + break + } + return cid.Undef, nil, err + } + + cids = append(cids, blk.Cid()) + + if err := ds.Put(ctx, blk); err != nil { + return cid.Undef, nil, err + } + } + + return carr.Header.Roots[0], ds, nil +} + +var zeroShard CarShard + +func (sqs *SQLiteStore) NewDeltaSession(ctx context.Context, user models.Uid, since *string) (*DeltaSession, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "NewSession") + defer span.End() + + // TODO: ensure that we don't write updates on top of the wrong head + // this needs to be a compare and swap type operation + lastShard, err := sqs.lastShardCache.get(ctx, user) + if err != nil { + return nil, fmt.Errorf("NewDeltaSession, lsc, %w", err) + } + + if lastShard == nil { + lastShard = &zeroShard + } + + if since != nil && *since != lastShard.Rev { + return nil, fmt.Errorf("revision mismatch: %s != %s: %w", *since, lastShard.Rev, ErrRepoBaseMismatch) + } + + return &DeltaSession{ + blks: make(map[cid.Cid]blockformat.Block), + base: &sqliteUserView{ + uid: user, + sqs: sqs, + }, + user: user, + baseCid: lastShard.Root.CID, + cs: sqs, + seq: lastShard.Seq + 1, + lastRev: lastShard.Rev, + }, nil +} + +func (sqs *SQLiteStore) ReadOnlySession(user models.Uid) (*DeltaSession, error) { + return &DeltaSession{ + base: &sqliteUserView{ + uid: user, + sqs: sqs, + }, + readonly: true, + user: user, + cs: sqs, + }, nil +} + +type cartmp struct { + xcid cid.Cid + rev string + root string + block []byte +} + +// ReadUserCar +// incremental is only ever called true +func (sqs *SQLiteStore) ReadUserCar(ctx context.Context, user models.Uid, sinceRev string, incremental bool, shardOut io.Writer) error { + sqGetCar.Inc() + ctx, span := otel.Tracer("carstore").Start(ctx, "ReadUserCar") + defer span.End() + + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return fmt.Errorf("rcar tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT cid,rev,root,block FROM blocks WHERE uid = ? AND rev > ? ORDER BY rev DESC") + if err != nil { + return fmt.Errorf("rcar sql, %w", err) + } + defer qstmt.Close() + rows, err := qstmt.QueryContext(ctx, user, sinceRev) + if err != nil { + return fmt.Errorf("rcar err, %w", err) + } + nblocks := 0 + first := true + for rows.Next() { + var xcid models.DbCID + var xrev string + var xroot models.DbCID + var xblock []byte + err = rows.Scan(&xcid, &xrev, &xroot, &xblock) + if err != nil { + return fmt.Errorf("rcar bad scan, %w", err) + } + if first { + if err := car.WriteHeader(&car.CarHeader{ + Roots: []cid.Cid{xroot.CID}, + Version: 1, + }, shardOut); err != nil { + return fmt.Errorf("rcar bad header, %w", err) + } + first = false + } + nblocks++ + _, err := LdWrite(shardOut, xcid.CID.Bytes(), xblock) + if err != nil { + return fmt.Errorf("rcar bad write, %w", err) + } + } + sqs.log.Debug("read car", "nblocks", nblocks, "since", sinceRev) + return nil +} + +// Stat is only used in a debugging admin handler +// don't bother implementing it (for now?) +func (sqs *SQLiteStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error) { + sqs.log.Warn("Stat debugging method not implemented for sqlite store") + return nil, nil +} + +func (sqs *SQLiteStore) WipeUserData(ctx context.Context, user models.Uid) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "WipeUserData") + defer span.End() + tx, err := sqs.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("wipe tx, %w", err) + } + defer tx.Rollback() + deleteResult, err := tx.ExecContext(ctx, "DELETE FROM blocks WHERE uid = ?", user) + nrows, ierr := deleteResult.RowsAffected() + if ierr == nil { + sqRowsDeleted.Add(float64(nrows)) + } + if err == nil { + err = ierr + } + if err == nil { + err = tx.Commit() + } + return err +} + +var txReadOnly = sql.TxOptions{ReadOnly: true} + +// HasUidCid needed for NewDeltaSession userView +func (sqs *SQLiteStore) HasUidCid(ctx context.Context, user models.Uid, bcid cid.Cid) (bool, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + sqHas.Inc() + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return false, fmt.Errorf("hasUC tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT rev, root FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") + if err != nil { + return false, fmt.Errorf("hasUC sql, %w", err) + } + defer qstmt.Close() + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{CID: bcid}) + if err != nil { + return false, fmt.Errorf("hasUC err, %w", err) + } + if rows.Next() { + var rev string + var rootb models.DbCID + err = rows.Scan(&rev, &rootb) + if err != nil { + return false, fmt.Errorf("hasUC bad scan, %w", err) + } + return true, nil + } + return false, nil +} + +func (sqs *SQLiteStore) CarStore() CarStore { + return sqs +} + +func (sqs *SQLiteStore) Close() error { + return sqs.db.Close() +} + +func (sqs *SQLiteStore) getBlock(ctx context.Context, user models.Uid, bcid cid.Cid) (blockformat.Block, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + sqGetBlock.Inc() + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return nil, fmt.Errorf("getb tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT block FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") + if err != nil { + return nil, fmt.Errorf("getb sql, %w", err) + } + defer qstmt.Close() + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{CID: bcid}) + if err != nil { + return nil, fmt.Errorf("getb err, %w", err) + } + if rows.Next() { + //var rev string + //var rootb models.DbCID + var blockb []byte + err = rows.Scan(&blockb) + if err != nil { + return nil, fmt.Errorf("getb bad scan, %w", err) + } + blk, err := blocks.NewBlockWithCid(blockb, bcid) + if err != nil { + return nil, fmt.Errorf("getb bad block, %w", err) + } + return blk, nil + } + return nil, ErrNothingThere +} + +func (sqs *SQLiteStore) getBlockSize(ctx context.Context, user models.Uid, bcid cid.Cid) (int64, error) { + // TODO: this is pretty cacheable? invalidate (uid,*) on WipeUserData + sqGetBlockSize.Inc() + tx, err := sqs.db.BeginTx(ctx, &txReadOnly) + if err != nil { + return 0, fmt.Errorf("getbs tx, %w", err) + } + defer tx.Rollback() + qstmt, err := tx.PrepareContext(ctx, "SELECT length(block) FROM blocks WHERE uid = ? AND cid = ? LIMIT 1") + if err != nil { + return 0, fmt.Errorf("getbs sql, %w", err) + } + defer qstmt.Close() + rows, err := qstmt.QueryContext(ctx, user, models.DbCID{CID: bcid}) + if err != nil { + return 0, fmt.Errorf("getbs err, %w", err) + } + if rows.Next() { + var out int64 + err = rows.Scan(&out) + if err != nil { + return 0, fmt.Errorf("getbs bad scan, %w", err) + } + return out, nil + } + return 0, nil +} + +type sqliteUserViewInner interface { + HasUidCid(ctx context.Context, user models.Uid, bcid cid.Cid) (bool, error) + getBlock(ctx context.Context, user models.Uid, bcid cid.Cid) (blockformat.Block, error) + getBlockSize(ctx context.Context, user models.Uid, bcid cid.Cid) (int64, error) +} + +// TODO: rename, used by both sqlite and scylla +type sqliteUserView struct { + sqs sqliteUserViewInner + uid models.Uid +} + +func (s sqliteUserView) Has(ctx context.Context, c cid.Cid) (bool, error) { + // TODO: cache block metadata? + return s.sqs.HasUidCid(ctx, s.uid, c) +} + +func (s sqliteUserView) Get(ctx context.Context, c cid.Cid) (blockformat.Block, error) { + // TODO: cache blocks? + return s.sqs.getBlock(ctx, s.uid, c) +} + +func (s sqliteUserView) GetSize(ctx context.Context, c cid.Cid) (int, error) { + // TODO: cache block metadata? + bigsize, err := s.sqs.getBlockSize(ctx, s.uid, c) + return int(bigsize), err +} + +// ensure we implement the interface +var _ minBlockstore = (*sqliteUserView)(nil) + +var sqRowsDeleted = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_rows_deleted", + Help: "User rows deleted in sqlite backend", +}) + +var sqGetBlock = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_get_block", + Help: "get block sqlite backend", +}) + +var sqGetBlockSize = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_get_block_size", + Help: "get block size sqlite backend", +}) + +var sqGetCar = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_get_car", + Help: "get block sqlite backend", +}) + +var sqHas = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_has", + Help: "check block presence sqlite backend", +}) + +var sqGetLastShard = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_get_last_shard", + Help: "get last shard sqlite backend", +}) + +var sqWriteNewShard = promauto.NewCounter(prometheus.CounterOpts{ + Name: "bgs_sq_write_shard", + Help: "write shard blocks sqlite backend", +}) diff --git a/carstore/util.go b/carstore/util.go new file mode 100644 index 000000000..56396501f --- /dev/null +++ b/carstore/util.go @@ -0,0 +1,32 @@ +package carstore + +import ( + "encoding/binary" + "io" +) + +// Length-delimited Write +// Writer stream gets Uvarint length then concatenated data +func LdWrite(w io.Writer, d ...[]byte) (int64, error) { + var sum uint64 + for _, s := range d { + sum += uint64(len(s)) + } + + buf := make([]byte, 8) + n := binary.PutUvarint(buf, sum) + nw, err := w.Write(buf[:n]) + if err != nil { + return 0, err + } + + for _, s := range d { + onw, err := w.Write(s) + if err != nil { + return int64(nw), err + } + nw += onw + } + + return int64(nw), nil +} diff --git a/cmd/astrolabe/README.md b/cmd/astrolabe/README.md new file mode 100644 index 000000000..7e738253f --- /dev/null +++ b/cmd/astrolabe/README.md @@ -0,0 +1,5 @@ + +astrolabe: basic atproto network data explorer +============================================== + +**NOTE: this proof-of-concept has moved to [cobalt](https://tangled.org/@bnewbold.net/cobalt/tree/main/cmd/astrolabe)** diff --git a/cmd/athome/README.md b/cmd/athome/README.md new file mode 100644 index 000000000..c22def967 --- /dev/null +++ b/cmd/athome/README.md @@ -0,0 +1,5 @@ + +athome: Public Bluesky Web Home +=============================== + +**NOTE: this proof-of-concept has moved to [cobalt](https://tangled.org/@bnewbold.net/cobalt/tree/main/cmd/athome)** diff --git a/cmd/beemo/Dockerfile b/cmd/beemo/Dockerfile new file mode 100644 index 000000000..2011ee103 --- /dev/null +++ b/cmd/beemo/Dockerfile @@ -0,0 +1,35 @@ +# Run this dockerfile from the top level of the indigo git repository like: +# +# podman build -f ./cmd/beemo/Dockerfile -t beemo . + +### Compile stage +FROM golang:1.25-alpine3.22 AS build-env +RUN apk add --no-cache build-base make git + +ADD . /dockerbuild +WORKDIR /dockerbuild + +# timezone data for alpine builds +ENV GOEXPERIMENT=loopvar +RUN GIT_VERSION=$(git describe --tags --long --always) && \ + go build -tags timetzdata -o /beemo ./cmd/beemo + +### Run stage +FROM alpine:3.22 + +RUN apk add --no-cache --update dumb-init ca-certificates runit +ENTRYPOINT ["dumb-init", "--"] + +WORKDIR / +RUN mkdir -p data/beemo +COPY --from=build-env /beemo / + +# small things to make golang binaries work well under alpine +ENV GODEBUG=netdns=go +ENV TZ=Etc/UTC + +CMD ["/beemo"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="ATP ChatOps Bot (beemo)" +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" diff --git a/cmd/beemo/README.md b/cmd/beemo/README.md new file mode 100644 index 000000000..4908395c5 --- /dev/null +++ b/cmd/beemo/README.md @@ -0,0 +1,17 @@ + +## beemo: Slack notification bot for moderation reports + +You need an admin token, slack webhook URL, and auth file (see gosky docs). +The auth file isn't actually used, only the admin token. + + # configure a slack webhook + export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T028K87/B04NBDB/oWbsHasdf23r2d + + # example pulling admin token out of `pass` password manager + export ATP_AUTH_ADMIN_PASSWORD=`pass bsky/pds-admin-staging | head -n1` + + # example just setting admin token directly + export ATP_AUTH_ADMIN_PASSWORD="someinsecurething123" + + # run the bot + GOLOG_LOG_LEVEL=debug go run ./cmd/beemo/ --pds https://pds.staging.example.com --auth bsky.auth notify-reports diff --git a/cmd/beemo/firehose_consumer.go b/cmd/beemo/firehose_consumer.go new file mode 100644 index 000000000..10216da73 --- /dev/null +++ b/cmd/beemo/firehose_consumer.go @@ -0,0 +1,126 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "net/http" + "net/url" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/schedulers/parallel" + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/repo" + "github.com/bluesky-social/indigo/repomgr" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/gorilla/websocket" +) + +func RunFirehoseConsumer(ctx context.Context, logger *slog.Logger, relayHost string, postCallback func(context.Context, syntax.DID, syntax.RecordKey, appbsky.FeedPost) error) error { + + dialer := websocket.DefaultDialer + u, err := url.Parse(relayHost) + if err != nil { + return fmt.Errorf("invalid relayHost URI: %w", err) + } + // always continue at the current cursor offset (don't provide cursor query param) + u.Path = "xrpc/com.atproto.sync.subscribeRepos" + logger.Info("subscribing to repo event stream", "upstream", relayHost) + con, _, err := dialer.Dial(u.String(), http.Header{ + "User-Agent": []string{fmt.Sprintf("beemo/%s", versioninfo.Short())}, + }) + if err != nil { + return fmt.Errorf("subscribing to firehose failed (dialing): %w", err) + } + + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + return HandleRepoCommit(ctx, logger, evt, postCallback) + }, + // NOTE: could add other callbacks as needed + } + + var scheduler events.Scheduler + // use parallel scheduler + parallelism := 4 + scheduler = parallel.NewScheduler( + parallelism, + 1000, + relayHost, + rsc.EventHandler, + ) + logger.Info("beemo firehose scheduler configured", "scheduler", "parallel", "workers", parallelism) + + return events.HandleRepoStream(ctx, con, scheduler, logger) +} + +// NOTE: for now, this function basically never errors, just logs and returns nil. Should think through error processing better. +func HandleRepoCommit(ctx context.Context, logger *slog.Logger, evt *comatproto.SyncSubscribeRepos_Commit, postCallback func(context.Context, syntax.DID, syntax.RecordKey, appbsky.FeedPost) error) error { + + logger = logger.With("event", "commit", "did", evt.Repo, "rev", evt.Rev, "seq", evt.Seq) + logger.Debug("received commit event") + + if evt.TooBig { + logger.Warn("skipping tooBig events for now") + return nil + } + + did, err := syntax.ParseDID(evt.Repo) + if err != nil { + logger.Error("bad DID syntax in event", "err", err) + return nil + } + + rr, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks)) + if err != nil { + logger.Error("failed to read repo from car", "err", err) + return nil + } + + for _, op := range evt.Ops { + logger = logger.With("eventKind", op.Action, "path", op.Path) + collection, rkey, err := syntax.ParseRepoPath(op.Path) + if err != nil { + logger.Error("invalid path in repo op") + return nil + } + + ek := repomgr.EventKind(op.Action) + switch ek { + case repomgr.EvtKindCreateRecord, repomgr.EvtKindUpdateRecord: + // read the record bytes from blocks, and verify CID + rc, recordCBOR, err := rr.GetRecordBytes(ctx, op.Path) + if err != nil { + logger.Error("reading record from event blocks (CAR)", "err", err) + continue + } + if op.Cid == nil || lexutil.LexLink(rc) != *op.Cid { + logger.Error("mismatch between commit op CID and record block", "recordCID", rc, "opCID", op.Cid) + continue + } + + switch collection { + case "app.bsky.feed.post": + var post appbsky.FeedPost + if err := post.UnmarshalCBOR(bytes.NewReader(*recordCBOR)); err != nil { + logger.Error("failed to parse app.bsky.feed.post record", "err", err) + continue + } + if err := postCallback(ctx, did, rkey, post); err != nil { + logger.Error("failed to process post record", "err", err) + continue + } + } + + default: + // ignore other events + } + } + + return nil +} diff --git a/cmd/beemo/main.go b/cmd/beemo/main.go new file mode 100644 index 000000000..1658b410b --- /dev/null +++ b/cmd/beemo/main.go @@ -0,0 +1,141 @@ +// Bluesky MOderation bot (BMO), a chatops helper for slack +// For now, polls a PDS for new moderation reports and publishes notifications to slack + +package main + +import ( + "context" + "io" + "log/slog" + "os" + "strings" + + _ "github.com/joho/godotenv/autoload" + _ "go.uber.org/automaxprocs" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/urfave/cli/v3" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("exiting", "err", err) + os.Exit(-1) + } +} + +func run(args []string) error { + + app := cli.Command{ + Name: "beemo", + Usage: "bluesky moderation reporting bot", + Version: versioninfo.Short(), + } + + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "log-level", + Usage: "log verbosity level (eg: warn, info, debug)", + Sources: cli.EnvVars("BEEMO_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"), + }, + &cli.StringFlag{ + Name: "slack-webhook-url", + // eg: https://hooks.slack.com/services/X1234 + Usage: "full URL of slack webhook", + Required: true, + Sources: cli.EnvVars("SLACK_WEBHOOK_URL"), + }, + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "notify-reports", + Usage: "watch for new moderation reports, notify in slack", + Action: pollNewReports, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "pds-host", + Usage: "method, hostname, and port of PDS instance", + Value: "http://localhost:4849", + Sources: cli.EnvVars("ATP_PDS_HOST"), + }, + &cli.StringFlag{ + Name: "admin-host", + Usage: "method, hostname, and port of admin interface (eg, Ozone), for direct links", + Value: "http://localhost:3000", + Sources: cli.EnvVars("ATP_ADMIN_HOST"), + }, + &cli.IntFlag{ + Name: "poll-period", + Usage: "API poll period in seconds", + Value: 30, + Sources: cli.EnvVars("POLL_PERIOD"), + }, + &cli.StringFlag{ + Name: "handle", + Usage: "for PDS login", + Required: true, + Sources: cli.EnvVars("ATP_AUTH_HANDLE"), + }, + &cli.StringFlag{ + Name: "password", + Usage: "for PDS login", + Required: true, + Sources: cli.EnvVars("ATP_AUTH_PASSWORD"), + }, + &cli.StringFlag{ + Name: "admin-password", + Usage: "admin authentication password for PDS", + Required: true, + Sources: cli.EnvVars("ATP_AUTH_ADMIN_PASSWORD"), + }, + }, + }, + &cli.Command{ + Name: "notify-mentions", + Usage: "watch firehose for posts mentioning specific accounts", + Action: notifyMentions, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "relay-host", + Usage: "method, hostname, and port of Relay instance (websocket)", + Value: "wss://bsky.network", + Sources: cli.EnvVars("ATP_RELAY_HOST"), + }, + &cli.StringFlag{ + Name: "mention-dids", + Usage: "DIDs to look for in mentions (comma-separated)", + Required: true, + Sources: cli.EnvVars("BEEMO_MENTION_DIDS"), + }, + &cli.IntFlag{ + Name: "minimum-words", + Usage: "minimum length of post text (word count; zero for no minimum)", + Value: 0, + Sources: cli.EnvVars("BEEMO_MINIMUM_WORDS"), + }, + }, + }, + } + return app.Run(context.Background(), args) +} + +func configLogger(cmd *cli.Command, writer io.Writer) *slog.Logger { + var level slog.Level + switch strings.ToLower(cmd.String("log-level")) { + case "error": + level = slog.LevelError + case "warn": + level = slog.LevelWarn + case "info": + level = slog.LevelInfo + case "debug": + level = slog.LevelDebug + default: + level = slog.LevelInfo + } + logger := slog.New(slog.NewJSONHandler(writer, &slog.HandlerOptions{ + Level: level, + })) + slog.SetDefault(logger) + return logger +} diff --git a/cmd/beemo/notify_mentions.go b/cmd/beemo/notify_mentions.go new file mode 100644 index 000000000..f1cba426e --- /dev/null +++ b/cmd/beemo/notify_mentions.go @@ -0,0 +1,97 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "strings" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/urfave/cli/v3" +) + +type MentionChecker struct { + slackWebhookURL string + mentionDIDs []syntax.DID + logger *slog.Logger + directory identity.Directory + minimumWords int +} + +func (mc *MentionChecker) ProcessPost(ctx context.Context, did syntax.DID, rkey syntax.RecordKey, post appbsky.FeedPost) error { + mc.logger.Debug("processing post record", "did", did, "rkey", rkey) + + if mc.minimumWords > 0 { + words := strings.Split(post.Text, " ") + if len(words) < mc.minimumWords { + return nil + } + } + + for _, facet := range post.Facets { + for _, feature := range facet.Features { + mention := feature.RichtextFacet_Mention + if mention == nil { + continue + } + for _, d := range mc.mentionDIDs { + if mention.Did == d.String() { + mc.logger.Info("found mention", "target", d, "author", did, "rkey", rkey) + targetIdent, err := mc.directory.LookupDID(ctx, syntax.DID(mention.Did)) + if err != nil { + return err + } + authorIdent, err := mc.directory.LookupDID(ctx, did) + if err != nil { + return err + } + msg := fmt.Sprintf("Mention of `@%s` by `@%s` ():\n```%s```", targetIdent.Handle, authorIdent.Handle, did, rkey, post.Text) + if post.Embed != nil && (post.Embed.EmbedImages != nil || post.Embed.EmbedRecordWithMedia != nil || post.Embed.EmbedRecord != nil || post.Embed.EmbedExternal != nil) { + msg += "\n(post also contains an embed/quote/media)" + } + return sendSlackMsg(ctx, msg, mc.slackWebhookURL) + } + } + } + } + return nil +} + +func notifyMentions(ctx context.Context, cmd *cli.Command) error { + logger := configLogger(cmd, os.Stdout) + relayHost := cmd.String("relay-host") + minimumWords := cmd.Int("minimum-words") + + mentionDIDs := []syntax.DID{} + for _, raw := range strings.Split(cmd.String("mention-dids"), ",") { + did, err := syntax.ParseDID(raw) + if err != nil { + return err + } + mentionDIDs = append(mentionDIDs, did) + } + + checker := MentionChecker{ + slackWebhookURL: cmd.String("slack-webhook-url"), + mentionDIDs: mentionDIDs, + logger: logger, + directory: identity.DefaultDirectory(), + minimumWords: minimumWords, + } + + logger.Info("beemo mention checker starting up...", "relayHost", relayHost, "mentionDIDs", mentionDIDs) + + // can flip this bool to false to prevent spamming slack channel on startup + if true { + err := sendSlackMsg(ctx, fmt.Sprintf("beemo booting, looking for account mentions: `%s`", mentionDIDs), checker.slackWebhookURL) + if err != nil { + return err + } + } + + return RunFirehoseConsumer(ctx, logger, relayHost, checker.ProcessPost) +} diff --git a/cmd/beemo/notify_reports.go b/cmd/beemo/notify_reports.go new file mode 100644 index 000000000..3769f39e5 --- /dev/null +++ b/cmd/beemo/notify_reports.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/urfave/cli/v3" +) + +func pollNewReports(ctx context.Context, cmd *cli.Command) error { + logger := configLogger(cmd, os.Stdout) + slackWebhookURL := cmd.String("slack-webhook-url") + + // record last-seen report timestamp + since := time.Now() + // NOTE: uncomment this for testing + //since = time.Now().Add(time.Duration(-12) * time.Hour) + period := time.Duration(cmd.Int("poll-period")) * time.Second + + // create a new session + xrpcc := &xrpc.Client{ + Client: util.RobustHTTPClient(), + Host: cmd.String("pds-host"), + Auth: &xrpc.AuthInfo{Handle: cmd.String("handle")}, + } + + auth, err := comatproto.ServerCreateSession(ctx, xrpcc, &comatproto.ServerCreateSession_Input{ + Identifier: xrpcc.Auth.Handle, + Password: cmd.String("password"), + }) + if err != nil { + return err + } + xrpcc.Auth.AccessJwt = auth.AccessJwt + xrpcc.Auth.RefreshJwt = auth.RefreshJwt + xrpcc.Auth.Did = auth.Did + xrpcc.Auth.Handle = auth.Handle + + adminToken := cmd.String("admin-password") + if len(adminToken) > 0 { + xrpcc.AdminToken = &adminToken + } + logger.Info("report polling bot starting up...") + // can flip this bool to false to prevent spamming slack channel on startup + if true { + err := sendSlackMsg(ctx, fmt.Sprintf("restarted bot, monitoring for reports since `%s`...", since.Format(time.RFC3339)), slackWebhookURL) + if err != nil { + return err + } + } + for { + // refresh session + xrpcc.Auth.AccessJwt = xrpcc.Auth.RefreshJwt + refresh, err := comatproto.ServerRefreshSession(ctx, xrpcc) + if err != nil { + return err + } + xrpcc.Auth.AccessJwt = refresh.AccessJwt + xrpcc.Auth.RefreshJwt = refresh.RefreshJwt + + // query just new reports (regardless of resolution state) + var limit int64 = 50 + me, err := toolsozone.ModerationQueryEvents( + ctx, + xrpcc, + nil, // addedLabels []string + nil, // addedTags []string + "", // ageAssuranceState + "", // batchId string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + "", // createdBy string + "", // cursor string + false, // hasComment bool + true, // includeAllUserRecords bool + limit, // limit int64 + nil, // modTool + nil, // policies []string + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + "", // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string + false, // withStrike bool + ) + if err != nil { + return err + } + // this works out to iterate from newest to oldest, which is the behavior we want (report only newest, then break) + for _, evt := range me.Events { + report := evt.Event.ModerationDefs_ModEventReport + // TODO: filter out based on subject state? similar to old "report.ResolvedByActionIds" + createdAt, err := time.Parse(time.RFC3339, evt.CreatedAt) + if err != nil { + return fmt.Errorf("invalid time format for 'createdAt': %w", err) + } + if createdAt.After(since) { + shortType := "" + if report.ReportType != nil && strings.Contains(*report.ReportType, "#") { + shortType = strings.SplitN(*report.ReportType, "#", 2)[1] + } + // ok, we found a "new" report, need to notify + msg := fmt.Sprintf("⚠️ New report at `%s` ⚠️\n", evt.CreatedAt) + msg += fmt.Sprintf("report id: `%d`\t", evt.Id) + msg += fmt.Sprintf("instance: `%s`\n", cmd.String("pds-host")) + msg += fmt.Sprintf("reasonType: `%s`\t", shortType) + msg += fmt.Sprintf("Admin: %s/reports/%d\n", cmd.String("admin-host"), evt.Id) + //msg += fmt.Sprintf("reportedByDid: `%s`\n", report.ReportedByDid) + logger.Info("found new report, notifying slack", "report", report) + err := sendSlackMsg(ctx, msg, slackWebhookURL) + if err != nil { + return fmt.Errorf("failed to send slack message: %w", err) + } + since = createdAt + break + } else { + logger.Debug("skipping report", "report", report) + } + } + logger.Info("... sleeping", "period", period) + time.Sleep(period) + } +} diff --git a/cmd/beemo/slack.go b/cmd/beemo/slack.go new file mode 100644 index 000000000..58887e72f --- /dev/null +++ b/cmd/beemo/slack.go @@ -0,0 +1,42 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/bluesky-social/indigo/util" +) + +type SlackWebhookBody struct { + Text string `json:"text"` +} + +// sends a simple slack message to a channel via "incoming webhook" +// The slack incoming webhook must be already configured in the slack workplace. +func sendSlackMsg(ctx context.Context, msg, webhookURL string) error { + // loosely based on: https://golangcode.com/send-slack-messages-without-a-library/ + + body, _ := json.Marshal(SlackWebhookBody{Text: msg}) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookURL, bytes.NewBuffer(body)) + if err != nil { + return err + } + req.Header.Add("Content-Type", "application/json") + client := util.RobustHTTPClient() + resp, err := client.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + if resp.StatusCode != 200 || buf.String() != "ok" { + return fmt.Errorf("failed slack webhook POST request. status=%d", resp.StatusCode) + } + return nil +} diff --git a/cmd/bigsky/Dockerfile b/cmd/bigsky/Dockerfile new file mode 100644 index 000000000..34402f81a --- /dev/null +++ b/cmd/bigsky/Dockerfile @@ -0,0 +1,51 @@ +# Run this dockerfile from the top level of the indigo git repository like: +# +# podman build -f ./cmd/bigsky/Dockerfile -t bigsky . + +### Compile stage +FROM golang:1.25-alpine3.22 AS build-env +RUN apk add --no-cache build-base make git + +ADD . /dockerbuild +WORKDIR /dockerbuild + +# timezone data for alpine builds +ENV GOEXPERIMENT=loopvar +RUN GIT_VERSION=$(git describe --tags --long --always) && \ + echo "replace github.com/gocql/gocql => github.com/scylladb/gocql v1.14.4" >> go.mod && \ + go mod tidy && \ + go build -tags timetzdata,scylla -o /bigsky ./cmd/bigsky + +### Build Frontend stage +FROM node:18-alpine as web-builder + +WORKDIR /app + +COPY ts/bgs-dash /app/ + +RUN yarn install --frozen-lockfile + +RUN yarn build + +### Run stage +FROM alpine:3.22 + +RUN apk add --no-cache --update dumb-init ca-certificates runit +ENTRYPOINT ["dumb-init", "--"] + +WORKDIR / +RUN mkdir -p data/bigsky +COPY --from=build-env /bigsky / +COPY --from=web-builder /app/dist/ public/ + +# small things to make golang binaries work well under alpine +ENV GODEBUG=netdns=go +ENV TZ=Etc/UTC + +EXPOSE 2470 + +CMD ["/bigsky"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="ATP Relay (aka BGS)" +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" diff --git a/cmd/bigsky/README.md b/cmd/bigsky/README.md new file mode 100644 index 000000000..73218e3c5 --- /dev/null +++ b/cmd/bigsky/README.md @@ -0,0 +1,339 @@ + +`bigsky`: atproto Relay Service +=============================== + +*NOTE: "Relays" used to be called "Big Graph Servers", or "BGS", which inspired the name "bigsky". Many variables and packages still reference "bgs"* + +This is the implementation of an atproto Relay which is running in the production network, written and operated by Bluesky. + +In atproto, a Relay subscribes to multiple PDS hosts and outputs a combined "firehose" event stream. Downstream services can subscribe to this single firehose a get all relevant events for the entire network, or a specific sub-graph of the network. The Relay maintains a mirror of repo data from all accounts on the upstream PDS instances, and verifies repo data structure integrity and identity signatures. It is agnostic to applications, and does not validate data against atproto Lexicon schemas. + +This Relay implementation is designed to subscribe to the entire global network. The current state of the codebase is informally expected to scale to around 20 million accounts in the network, and thousands of repo events per second (peak). + +Features and design decisions: + +- runs on a single server +- repo data: stored on-disk in individual CAR "slice" files, with metadata in SQL. filesystem must accommodate tens of millions of small files +- firehose backfill data: stored on-disk by default, with metadata in SQL +- crawling and account state: stored in SQL database +- SQL driver: gorm, with PostgreSQL in production and sqlite for testing +- disk I/O intensive: fast NVMe disks are recommended, and RAM is helpful for caching +- highly concurrent: not particularly CPU intensive +- single golang binary for easy deployment +- observability: logging, prometheus metrics, OTEL traces +- "spidering" feature to auto-discover new accounts (DIDs) +- ability to export/import lists of DIDs to "backfill" Relay instances +- periodic repo compaction +- admin web interface: configure limits, add upstream PDS instances, etc + +This software is not as packaged, documented, and supported for self-hosting as our PDS distribution or Ozone service. But it is relatively simple and inexpensive to get running. + +A note and reminder about Relays in general are that they are more of a convenience in the protocol than a hard requirement. The "firehose" API is the exact same on the PDS and on a Relay. Any service which subscribes to the Relay could instead connect to one or more PDS instances directly. + + +## Development Tips + +The README and Makefile at the top level of this git repo have some generic helpers for testing, linting, formatting code, etc. + +To re-build and run the bigsky Relay locally: + + make run-dev-relay + +You can re-build and run the command directly to get a list of configuration flags and env vars; env vars will be loaded from `.env` if that file exists: + + RELAY_ADMIN_KEY=localdev go run ./cmd/bigsky/ --help + +By default, the daemon will use sqlite for databases (in the directory `./data/bigsky/`), CAR data will be stored as individual shard files in `./data/bigsky/carstore/`), and the HTTP API will be bound to localhost port 2470. + +When the daemon isn't running, sqlite database files can be inspected with: + + sqlite3 data/bigsky/bgs.sqlite + [...] + sqlite> .schema + +Wipe all local data: + + # careful! double-check this destructive command + rm -rf ./data/bigsky/* + +There is a basic web dashboard, though it will not be included unless built and copied to a local directory `./public/`. Run `make build-relay-ui`, and then when running the daemon the dashboard will be available at: . Paste in the admin key, eg `localdev`. + +The local admin routes can also be accessed by passing the admin key as a bearer token, for example: + + http get :2470/admin/pds/list Authorization:"Bearer localdev" + +Request crawl of an individual PDS instance like: + + http post :2470/admin/pds/requestCrawl Authorization:"Bearer localdev" hostname=pds.example.com + + +## Docker Containers + +One way to deploy is running a docker image. You can pull and/or run a specific version of bigsky, referenced by git commit, from the Bluesky Github container registry. For example: + + docker pull ghcr.io/bluesky-social/indigo:bigsky-fd66f93ce1412a3678a1dd3e6d53320b725978a6 + docker run ghcr.io/bluesky-social/indigo:bigsky-fd66f93ce1412a3678a1dd3e6d53320b725978a6 + +There is a Dockerfile in this directory, which can be used to build customized/patched versions of the Relay as a container, republish them, run locally, deploy to servers, deploy to an orchestrated cluster, etc. See docs and guides for docker and cluster management systems for details. + + +## Database Setup + +PostgreSQL and Sqlite are both supported. When using Sqlite, separate files are used for Relay metadata and CarStore metadata. With PostgreSQL a single database server, user, and logical database can all be reused: table names will not conflict. + +Database configuration is passed via the `DATABASE_URL` and `CARSTORE_DATABASE_URL` environment variables, or the corresponding CLI args. + +For PostgreSQL, the user and database must already be configured. Some example SQL commands are: + + CREATE DATABASE bgs; + CREATE DATABASE carstore; + + CREATE USER ${username} WITH PASSWORD '${password}'; + GRANT ALL PRIVILEGES ON DATABASE bgs TO ${username}; + GRANT ALL PRIVILEGES ON DATABASE carstore TO ${username}; + +This service currently uses `gorm` to automatically run database migrations as the regular user. There is no concept of running a separate set of migrations under more privileged database user. + + +## Deployment + +*NOTE: this is not a complete guide to operating a Relay. There are decisions to be made and communicated about policies, bandwidth use, PDS crawling and rate-limits, financial sustainability, etc, which are not covered here. This is just a quick overview of how to technically get a relay up and running.* + +In a real-world system, you will probably want to use PostgreSQL for both the relay database and the carstore database. CAR shards will still be stored on-disk, resulting in many millions of files. Chose your storage hardware and filesystem carefully: we recommend XFS on local NVMe, not network-backed blockstorage (eg, not EBS volumes on AWS). + +Some notable configuration env vars to set: + +- `ENVIRONMENT`: eg, `production` +- `DATABASE_URL`: see section below +- `CARSTORE_DATABASE_URL`: see section below +- `DATA_DIR`: CAR shards will be stored in a subdirectory +- `GOLOG_LOG_LEVEL`: log verbosity +- `RESOLVE_ADDRESS`: DNS server to use +- `FORCE_DNS_UDP`: recommend "true" +- `BGS_COMPACT_INTERVAL`: to control CAR compaction scheduling. for example, "8h" (every 8 hours). Set to "0" to disable automatic compaction. +- `MAX_CARSTORE_CONNECTIONS` and `MAX_METADB_CONNECTIONS`: number of concurrent SQL database connections +- `MAX_FETCH_CONCURRENCY`: how many outbound CAR backfill requests to make in parallel + +There is a health check endpoint at `/xrpc/_health`. Prometheus metrics are exposed by default on port 2471, path `/metrics`. The service logs fairly verbosely to stderr; use `GOLOG_LOG_LEVEL` to control log volume. + +As a rough guideline for the compute resources needed to run a full-network Relay, in June 2024 an example Relay for over 5 million repositories used: + +- around 30 million inodes (files) +- roughly 1 TByte of disk for PostgreSQL +- roughly 1 TByte of disk for CAR shard storage +- roughly 5k disk I/O operations per second (all combined) +- roughly 100% of one CPU core (quite low CPU utilization) +- roughly 5GB of RAM for bigsky, and as much RAM as available for PostgreSQL and page cache +- on the order of 1 megabit inbound bandwidth (crawling PDS instances) and 1 megabit outbound per connected client. 1 mbit continuous is approximately 350 GByte/month + +Be sure to double-check bandwidth usage and pricing if running a public relay! Bandwidth prices can vary widely between providers, and popular cloud services (AWS, Google Cloud, Azure) are very expensive compared to alternatives like OVH or Hetzner. + + +## Bootstrapping the Network + +To bootstrap the entire network, you'll want to start with a list of large PDS instances to backfill from. You could pull from a public dashboard of instances (like [mackuba's](https://blue.mackuba.eu/directory/pdses)), or scrape the full DID PLC directory, parse out all PDS service declarations, and sort by count. + +Once you have a set of PDS hosts, you can put the bare hostnames (not URLs: no `https://` prefix, port, or path suffix) in a `hosts.txt` file, and then use the `crawl_pds.sh` script to backfill and configure limits for all of them: + + export RELAY_HOST=your.pds.hostname.tld + export RELAY_ADMIN_KEY=your-secret-key + + # both request crawl, and set generous crawl limits for each + cat hosts.txt | parallel -j1 ./crawl_pds.sh {} + +Just consuming from the firehose for a few hours will only backfill accounts with activity during that period. This is fine to get the backfill process started, but eventually you'll want to do full "resync" of all the repositories on the PDS host to the most recent repo rev version. To enqueue that for all the PDS instances: + + # start sync/backfill of all accounts + cat hosts.txt | parallel -j1 ./sync_pds.sh {} + +Lastly, can monitor progress of any ongoing re-syncs: + + # check sync progress for all hosts + cat hosts.txt | parallel -j1 ./sync_pds.sh {} + + +## Admin API + +The relay has a number of admin HTTP API endpoints. Given a relay setup listening on port 2470 and with a reasonably secure admin secret: + +``` +RELAY_ADMIN_PASSWORD=$(openssl rand --hex 16) +bigsky --api-listen :2470 --admin-key ${RELAY_ADMIN_PASSWORD} ... +``` + +One can, for example, begin compaction of all repos + +``` +curl -H 'Authorization: Bearer '${RELAY_ADMIN_PASSWORD} -H 'Content-Type: application/x-www-form-urlencoded' --data '' http://127.0.0.1:2470/admin/repo/compactAll +``` + +### /admin/subs/getUpstreamConns + +Return list of PDS host names in json array of strings: ["host", ...] + +### /admin/subs/perDayLimit + +Return `{"limit": int}` for the number of new PDS subscriptions that the relay may start in a rolling 24 hour window. + +### /admin/subs/setPerDayLimit + +POST with `?limit={int}` to set the number of new PDS subscriptions that the relay may start in a rolling 24 hour window. + +### /admin/subs/setEnabled + +POST with param `?enabled=true` or `?enabled=false` to enable or disable PDS-requested new-PDS crawling. + +### /admin/subs/getEnabled + +Return `{"enabled": bool}` if non-admin new PDS crawl requests are enabled + +### /admin/subs/killUpstream + +POST with `?host={pds host name}` to disconnect from their firehose. + +Optionally add `&block=true` to prevent connecting to them in the future. + +### /admin/subs/listDomainBans + +Return `{"banned_domains": ["host name", ...]}` + +### /admin/subs/banDomain + +POST `{"Domain": "host name"}` to ban a domain + +### /admin/subs/unbanDomain + +POST `{"Domain": "host name"}` to un-ban a domain + +### /admin/repo/takeDown + +POST `{"did": "did:..."}` to take-down a bad repo; deletes all local data for the repo + +### /admin/repo/reverseTakedown + +POST `?did={did:...}` to reverse a repo take-down + +### /admin/repo/compact + +POST `?did={did:...}` to compact a repo. Optionally `&fast=true`. HTTP blocks until the compaction finishes. + +### /admin/repo/compactAll + +POST to begin compaction of all repos. Optional query params: + + * `fast=true` + * `limit={int}` maximum number of repos to compact (biggest first) (default 50) + * `threhsold={int}` minimum number of shard files a repo must have on disk to merit compaction (default 20) + +### /admin/repo/reset + +POST `?did={did:...}` deletes all local data for the repo + +### /admin/repo/verify + +POST `?did={did:...}` checks that all repo data is accessible. HTTP blocks until done. + +### /admin/pds/requestCrawl + +POST `{"hostname":"pds host"}` to start crawling a PDS + +### /admin/pds/list + +GET returns JSON list of records +```json +[{ + "Host": string, + "Did": string, + "SSL": bool, + "Cursor": int, + "Registered": bool, + "Blocked": bool, + "RateLimit": float, + "CrawlRateLimit": float, + "RepoCount": int, + "RepoLimit": int, + "HourlyEventLimit": int, + "DailyEventLimit": int, + + "HasActiveConnection": bool, + "EventsSeenSinceStartup": int, + "PerSecondEventRate": {"Max": float, "Window": float seconds}, + "PerHourEventRate": {"Max": float, "Window": float seconds}, + "PerDayEventRate": {"Max": float, "Window": float seconds}, + "CrawlRate": {"Max": float, "Window": float seconds}, + "UserCount": int, +}, ...] +``` + +### /admin/pds/resync + +POST `?host={host}` to start a resync of a PDS + +GET `?host={host}` to get status of a PDS resync, return + +```json +{"resync": { + "pds": { + "Host": string, + "Did": string, + "SSL": bool, + "Cursor": int, + "Registered": bool, + "Blocked": bool, + "RateLimit": float, + "CrawlRateLimit": float, + "RepoCount": int, + "RepoLimit": int, + "HourlyEventLimit": int, + "DailyEventLimit": int, + }, + "numRepoPages": int, + "numRepos": int, + "numReposChecked": int, + "numReposToResync": int, + "status": string, + "statusChangedAt": time, +}} +``` + +### /admin/pds/changeLimits + +POST to set the limits for a PDS. body: + +```json +{ + "host": string, + "per_second": int, + "per_hour": int, + "per_day": int, + "crawl_rate": int, + "repo_limit": int, +} +``` + +### /admin/pds/block + +POST `?host={host}` to block a PDS + +### /admin/pds/unblock + +POST `?host={host}` to un-block a PDS + + +### /admin/pds/addTrustedDomain + +POST `?domain={}` to make a domain trusted + +### /admin/consumers/list + +GET returns list json of clients currently reading from the relay firehose + +```json +[{ + "id": int, + "remote_addr": string, + "user_agent": string, + "events_consumed": int, + "connected_at": time, +}, ...] +``` diff --git a/cmd/bigsky/copy_pdses.py b/cmd/bigsky/copy_pdses.py new file mode 100644 index 000000000..093f8f1e3 --- /dev/null +++ b/cmd/bigsky/copy_pdses.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# +# pip install requests +# +# python3 copy_pdses.py --admin-key hunter2 --source-url http://srcrelay:2470 --dest-url http://destrelay:2470 + +import json +import logging +import sys +import urllib.parse + +import requests + +logger = logging.getLogger(__name__) + +class relay: + def __init__(self, rooturl, headers=None, session=None): + "rooturl string, headers dict or None, session requests.Session() or None" + self.rooturl = rooturl + self.headers = headers or dict() + self.session = session or requests.Session() + + def crawl(self, host): + pheaders = dict(self.headers) + pheaders['Content-Type'] = 'application/json' + url = urllib.parse.urljoin(self.rooturl, '/admin/pds/requestCrawl') + response = self.session.post(url, headers=pheaders, data=json.dumps({"hostname": host})) + if response.status_code != 200: + return False + return True + + def crawlAndSetLimits(self, host, limits): + "host string, limits dict" + if not self.crawl(host): + logger.error("%s %s : %d %r", url, host, response.status_code, response.text) + return + if limits is None: + logger.debug("requestCrawl %s OK", host) + if self.setLimits(host, limits): + logger.debug("requestCrawl + changeLimits %s OK", host) + def setLimits(self, host, limits): + url = urllib.parse.urljoin(self.rooturl, '/admin/pds/changeLimits') + plimits = dict(limits) + plimits["host"] = host + pheaders = dict(self.headers) + pheaders['Content-Type'] = 'application/json' + response = self.session.post(url, headers=pheaders, data=json.dumps(plimits)) + if response.status_code != 200: + logger.error("%s %s : %d %r", url, host, response.status_code, response.text) + return False + return True + + def crawlAndBlock(self, host): + "make relay aware of PDS, and block it" + if not self.crawl(host): + logger.error("%s %s : %d %r", url, host, response.status_code, response.text) + return + if self.block(host): + logger.debug("requestCrawl + block %s OK", host) + + def block(self, host): + url = urllib.parse.urljoin(self.rooturl, '/admin/pds/block') + response = self.session.post(url, headers=self.headers, data='', params={"host":host}) + if response.status_code != 200: + logger.error("%s %s : %d %r", url, host, response.status_code, response.text) + return False + return True + + def unblock(self, host): + url = urllib.parse.urljoin(self.rooturl, '/admin/pds/unblock') + response = self.session.post(url, headers=self.headers, data='', params={"host":host}) + if response.status_code != 200: + logger.error("%s %s : %d %r", url, host, response.status_code, response.text) + return False + return True + + def pdsList(self): + "GET /admin/pds/list" + url = urllib.parse.urljoin(self.rooturl, '/admin/pds/list') + response = self.session.get(url, headers=self.headers) + if response.status_code != 200: + logger.error("%s : %d %r", url, response.status_code, response.text) + return None + return response.json() + +def makeByHost(they): + out = dict() + for rec in they: + out[rec['Host']] = rec + return out + +def makeLimits(rec): + "for submitting to changeLimits" + return { + "host": rec['Host'], + "per_second":rec['RateLimit'], + "per_hour":rec['HourlyEventLimit'], + "per_day":rec['DailyEventLimit'], + "crawl_rate":rec['CrawlRateLimit'], + "repo_limit":rec['RepoLimit'], + } + +def makeRequestCrawl(rec): + "for submitting to requestCrawl" + return {"hostname":rec["Host"]} + +def de(a,b): + # dict equal + for ka, va in a.items(): + vb = b[ka] + if (va is None) and (vb is None): + continue + if va == vb: + continue + return False + for kb in b.keys(): + if kb not in a: + return False + return True + +def main(): + import argparse + ap = argparse.ArgumentParser() + ap.add_argument('--admin-key', default=None, help='relay auth bearer token', required=True) + ap.add_argument('--source-url', default=None, help='base url to GET /admin/pds/list') + ap.add_argument('--source-json', default=None, help='load /admin/pds/list json from file') + ap.add_argument('--dest-url', default=None, help='dest URL to POST requestCrawl etc to') + ap.add_argument('--dry-run', default=False, action='store_true') + ap.add_argument('--verbose', default=False, action='store_true') + args = ap.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + headers = {'Authorization': 'Bearer ' + args.admin_key} + + if args.source_json: + with open(args.source_json, 'rt') as fin: + sourceList = json.load(fin) + elif args.source_url: + relaySession = relay(args.source_url, headers) + sourceList = relaySession.pdsList() + else: + sys.stdout.write("need --source-url or --source-json\n") + sys.exit(1) + + r2 = relay(args.dest_url, headers) + destList = r2.pdsList() + + source = makeByHost(sourceList) + dests = makeByHost(destList) + + snotd = [] + dnots = [] + diflim = [] + difblock = [] + recrawl = [] + + for k1, v1 in source.items(): + v2 = dests.get(k1) + if v2 is None: + snotd.append(v1) + continue + lim1 = makeLimits(v1) + lim2 = makeLimits(v2) + if v1["Blocked"] != v2["Blocked"]: + difblock.append((k1,v1["Blocked"])) + if v1["Blocked"]: + continue + if not de(lim1, lim2): + diflim.append(lim1) + if v1["HasActiveConnection"] and not v2["HasActiveConnection"]: + recrawl.append(k1) + for k2 in dests.keys(): + if k2 not in source: + dnots.append(k2) + + logger.debug("%d source not dest", len(snotd)) + for rec in snotd: + if rec["Blocked"]: + if args.dry_run: + sys.stdout.write("crawl and block: {!r}\n".format(rec["Host"])) + else: + r2.crawlAndBlock(rec["Host"]) + else: + limits = makeLimits(rec) + if args.dry_run: + sys.stdout.write("crawl and limit: {}\n".format(json.dumps(limits))) + else: + r2.crawlAndSetLimits(rec["Host"], limits) + logger.debug("adjust limits: %d", len(diflim)) + for limits in diflim: + if args.dry_run: + sys.stdout.write("set limits: {}\n".format(json.dumps(limits))) + else: + r2.setLimits(limits["host"], limits) + logger.debug("adjust block status: %d", len(difblock)) + for host, blocked in difblock: + if args.dry_run: + sys.stdout.write("{} block={}\n".format(host, blocked)) + else: + if blocked: + r2.block(host) + else: + r2.unblock(host) + logger.debug("restart requestCrawl: %d", len(recrawl)) + for host in recrawl: + if args.dry_run: + logger.info("requestCrawl %s", host) + else: + if r2.crawl(host): + logger.debug("requestCrawl %s OK", host) + logger.info("%d in dest but not source", len(dnots)) + for k2 in dnots: + logger.debug("%s", k2) + + + + + +if __name__ == '__main__': + main() diff --git a/cmd/bigsky/crawl_pds.sh b/cmd/bigsky/crawl_pds.sh new file mode 100755 index 000000000..e55c2f5f4 --- /dev/null +++ b/cmd/bigsky/crawl_pds.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -e # fail on error +set -u # fail if variable not set in substitution +set -o pipefail # fail if part of a '|' command fails + +if test -z "${RELAY_ADMIN_KEY}"; then + echo "RELAY_ADMIN_KEY secret is not defined" + exit -1 +fi + +if test -z "${RELAY_HOST}"; then + echo "RELAY_HOST config not defined" + exit -1 +fi + +if test -z "$1"; then + echo "expected PDS hostname as an argument" + exit -1 +fi + +echo "requestCrawl $1" +http --quiet --ignore-stdin post https://${RELAY_HOST}/admin/pds/requestCrawl Authorization:"Bearer ${RELAY_ADMIN_KEY}" \ + hostname=$1 + +echo "changeLimits $1" +http --quiet --ignore-stdin post https://${RELAY_HOST}/admin/pds/changeLimits Authorization:"Bearer ${RELAY_ADMIN_KEY}" \ + per_second:=100 \ + per_hour:=1000000 \ + per_day:=1000000 \ + crawl_rate:=10 \ + repo_limit:=1000000 \ + host=$1 diff --git a/cmd/bigsky/docker-compose.yml b/cmd/bigsky/docker-compose.yml new file mode 100644 index 000000000..d4896f796 --- /dev/null +++ b/cmd/bigsky/docker-compose.yml @@ -0,0 +1,15 @@ +version: "3.8" +services: + postgres: + image: "postgres:14" + ports: + - "5432:5432" + volumes: + - type: bind + source: /mnt/postgres + target: /var/lib/postgresql/data + restart: always + environment: + POSTGRES_USER: bgs + POSTGRES_PASSWORD: 33pAstcHDMszLedQah2EVYNgnxbCP + POSTGRES_DB: bgs diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index 960331fa7..b6f0296e0 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -2,135 +2,569 @@ package main import ( "context" + "fmt" + "log/slog" + "net/http" + "net/url" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" - "github.com/bluesky-social/indigo/api" - "github.com/bluesky-social/indigo/bgs" + _ "github.com/joho/godotenv/autoload" + _ "go.uber.org/automaxprocs" + _ "net/http/pprof" + + libbgs "github.com/bluesky-social/indigo/bgs" "github.com/bluesky-social/indigo/carstore" - cliutil "github.com/bluesky-social/indigo/cmd/gosky/util" + "github.com/bluesky-social/indigo/did" "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/dbpersist" + "github.com/bluesky-social/indigo/events/diskpersist" + "github.com/bluesky-social/indigo/handles" "github.com/bluesky-social/indigo/indexer" - "github.com/bluesky-social/indigo/notifs" + "github.com/bluesky-social/indigo/plc" "github.com/bluesky-social/indigo/repomgr" - logging "github.com/ipfs/go-log" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/util/cliutil" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/earthboundkid/versioninfo/v2" "github.com/urfave/cli/v2" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/sdk/resource" tracesdk "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" "gorm.io/plugin/opentelemetry/tracing" ) -var log = logging.Logger("bgs") +var log = slog.Default().With("system", "bigsky") func init() { - logging.SetAllLoggers(logging.LevelDebug) + // control log level using, eg, GOLOG_LOG_LEVEL=debug + //logging.SetAllLoggers(logging.LevelDebug) } func main() { - app := cli.NewApp() + if err := run(os.Args); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } +} + +func run(args []string) error { + + app := cli.App{ + Name: "bigsky", + Usage: "atproto Relay daemon", + Version: versioninfo.Short(), + } app.Flags = []cli.Flag{ &cli.BoolFlag{ Name: "jaeger", }, &cli.StringFlag{ - Name: "db", - Value: "sqlite=bgs.db", + Name: "db-url", + Usage: "database connection string for BGS database", + Value: "sqlite://./data/bigsky/bgs.sqlite", + EnvVars: []string{"DATABASE_URL"}, }, &cli.StringFlag{ - Name: "carstoredb", - Value: "sqlite=carstore.db", + Name: "carstore-db-url", + Usage: "database connection string for carstore database", + Value: "sqlite://./data/bigsky/carstore.sqlite", + EnvVars: []string{"CARSTORE_DATABASE_URL"}, + }, + &cli.BoolFlag{ + Name: "db-tracing", }, &cli.StringFlag{ - Name: "carstore", - Value: "bgscarstore", + Name: "data-dir", + Usage: "path of directory for CAR files and other data", + Value: "data/bigsky", + EnvVars: []string{"RELAY_DATA_DIR", "DATA_DIR"}, + }, + &cli.StringFlag{ + Name: "plc-host", + Usage: "method, hostname, and port of PLC registry", + Value: "https://plc.directory", + EnvVars: []string{"ATP_PLC_HOST"}, }, &cli.BoolFlag{ - Name: "dbtracing", + Name: "crawl-insecure-ws", + Usage: "when connecting to PDS instances, use ws:// instead of wss://", + }, + &cli.StringFlag{ + Name: "api-listen", + Value: ":2470", + }, + &cli.StringFlag{ + Name: "metrics-listen", + Value: ":2471", + EnvVars: []string{"RELAY_METRICS_LISTEN", "BGS_METRICS_LISTEN"}, + }, + &cli.StringFlag{ + Name: "disk-persister-dir", + Usage: "set directory for disk persister (implicitly enables disk persister)", + EnvVars: []string{"RELAY_PERSISTER_DIR"}, }, &cli.StringFlag{ - Name: "plc", - Usage: "hostname of the plc server", - Value: "https://plc.directory", + Name: "admin-key", + EnvVars: []string{"RELAY_ADMIN_KEY", "BGS_ADMIN_KEY"}, + }, + &cli.StringSliceFlag{ + Name: "handle-resolver-hosts", + EnvVars: []string{"HANDLE_RESOLVER_HOSTS"}, + }, + &cli.IntFlag{ + Name: "max-carstore-connections", + EnvVars: []string{"MAX_CARSTORE_CONNECTIONS"}, + Value: 40, + }, + &cli.IntFlag{ + Name: "max-metadb-connections", + EnvVars: []string{"MAX_METADB_CONNECTIONS"}, + Value: 40, + }, + &cli.DurationFlag{ + Name: "compact-interval", + EnvVars: []string{"RELAY_COMPACT_INTERVAL", "BGS_COMPACT_INTERVAL"}, + Value: 4 * time.Hour, + Usage: "interval between compaction runs, set to 0 to disable scheduled compaction", + }, + &cli.StringFlag{ + Name: "resolve-address", + EnvVars: []string{"RESOLVE_ADDRESS"}, + Value: "1.1.1.1:53", + }, + &cli.BoolFlag{ + Name: "force-dns-udp", + EnvVars: []string{"FORCE_DNS_UDP"}, + }, + &cli.IntFlag{ + Name: "max-fetch-concurrency", + Value: 100, + EnvVars: []string{"MAX_FETCH_CONCURRENCY"}, + }, + &cli.StringFlag{ + Name: "env", + Value: "dev", + EnvVars: []string{"ENVIRONMENT"}, + Usage: "declared hosting environment (prod, qa, etc); used in metrics", + }, + &cli.StringFlag{ + Name: "otel-exporter-otlp-endpoint", + EnvVars: []string{"OTEL_EXPORTER_OTLP_ENDPOINT"}, + }, + &cli.StringFlag{ + Name: "bsky-social-rate-limit-skip", + EnvVars: []string{"BSKY_SOCIAL_RATE_LIMIT_SKIP"}, + Usage: "ratelimit bypass secret token for *.bsky.social domains", + }, + &cli.IntFlag{ + Name: "default-repo-limit", + Value: 100, + EnvVars: []string{"RELAY_DEFAULT_REPO_LIMIT"}, + }, + &cli.IntFlag{ + Name: "concurrency-per-pds", + EnvVars: []string{"RELAY_CONCURRENCY_PER_PDS"}, + Value: 100, + }, + &cli.IntFlag{ + Name: "max-queue-per-pds", + EnvVars: []string{"RELAY_MAX_QUEUE_PER_PDS"}, + Value: 1_000, + }, + &cli.IntFlag{ + Name: "did-cache-size", + EnvVars: []string{"RELAY_DID_CACHE_SIZE"}, + Value: 5_000_000, + }, + &cli.StringSliceFlag{ + Name: "did-memcached", + EnvVars: []string{"RELAY_DID_MEMCACHED"}, + }, + &cli.DurationFlag{ + Name: "event-playback-ttl", + Usage: "time to live for event playback buffering (only applies to disk persister)", + EnvVars: []string{"RELAY_EVENT_PLAYBACK_TTL"}, + Value: 72 * time.Hour, + }, + &cli.IntFlag{ + Name: "num-compaction-workers", + EnvVars: []string{"RELAY_NUM_COMPACTION_WORKERS"}, + Value: 2, + }, + &cli.StringSliceFlag{ + Name: "carstore-shard-dirs", + Usage: "specify list of shard directories for carstore storage, overrides default storage within datadir", + EnvVars: []string{"RELAY_CARSTORE_SHARD_DIRS"}, + }, + &cli.StringSliceFlag{ + Name: "next-crawler", + Usage: "forward POST requestCrawl to this url, should be machine root url and not xrpc/requestCrawl, comma separated list", + EnvVars: []string{"RELAY_NEXT_CRAWLER"}, + }, + &cli.BoolFlag{ + Name: "ex-sqlite-carstore", + Usage: "enable experimental sqlite carstore", + Value: false, + }, + &cli.StringSliceFlag{ + Name: "scylla-carstore", + Usage: "scylla server addresses for storage backend, comma separated", + Value: &cli.StringSlice{}, + EnvVars: []string{"RELAY_SCYLLA_NODES"}, + }, + &cli.BoolFlag{ + Name: "non-archival", + EnvVars: []string{"RELAY_NON_ARCHIVAL"}, + Value: false, }, } - app.Action = func(cctx *cli.Context) error { + app.Action = runBigsky + return app.Run(os.Args) +} - if cctx.Bool("jaeger") { - url := "http://localhost:14268/api/traces" - exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url))) - if err != nil { - return err - } - tp := tracesdk.NewTracerProvider( - // Always be sure to batch in production. - tracesdk.WithBatcher(exp), - // Record information about this application in a Resource. - tracesdk.WithResource(resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String("bgs"), - attribute.String("environment", "test"), - attribute.Int64("ID", 1), - )), - ) +func setupOTEL(cctx *cli.Context) error { - otel.SetTracerProvider(tp) + env := cctx.String("env") + if env == "" { + env = "dev" + } + if cctx.Bool("jaeger") { + jaegerUrl := "http://localhost:14268/api/traces" + exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerUrl))) + if err != nil { + return err } + tp := tracesdk.NewTracerProvider( + // Always be sure to batch in production. + tracesdk.WithBatcher(exp), + // Record information about this application in a Resource. + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("bgs"), + attribute.String("env", env), // DataDog + attribute.String("environment", env), // Others + attribute.Int64("ID", 1), + )), + ) + + otel.SetTracerProvider(tp) + } - dbstr := cctx.String("db") + // Enable OTLP HTTP exporter + // For relevant environment variables: + // https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables + // At a minimum, you need to set + // OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 + if ep := cctx.String("otel-exporter-otlp-endpoint"); ep != "" { + slog.Info("setting up trace exporter", "endpoint", ep) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - db, err := cliutil.SetupDatabase(dbstr) + exp, err := otlptracehttp.New(ctx) if err != nil { + slog.Error("failed to create trace exporter", "error", err) + os.Exit(1) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := exp.Shutdown(ctx); err != nil { + slog.Error("failed to shutdown trace exporter", "error", err) + } + }() + + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exp), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("bgs"), + attribute.String("env", env), // DataDog + attribute.String("environment", env), // Others + attribute.Int64("ID", 1), + )), + ) + otel.SetTracerProvider(tp) + } + + return nil +} + +func runBigsky(cctx *cli.Context) error { + // Trap SIGINT to trigger a shutdown. + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + _, _, err := cliutil.SetupSlog(cliutil.LogOptions{}) + if err != nil { + return err + } + + // start observability/tracing (OTEL and jaeger) + if err := setupOTEL(cctx); err != nil { + return err + } + + // ensure data directory exists; won't error if it does + datadir := cctx.String("data-dir") + csdir := filepath.Join(datadir, "carstore") + if err := os.MkdirAll(datadir, os.ModePerm); err != nil { + return err + } + + dburl := cctx.String("db-url") + slog.Info("setting up main database", "url", dburl) + db, err := cliutil.SetupDatabase(dburl, cctx.Int("max-metadb-connections")) + if err != nil { + return err + } + if cctx.Bool("db-tracing") { + if err := db.Use(tracing.NewPlugin()); err != nil { return err } + } - if cctx.Bool("dbtracing") { - if err := db.Use(tracing.NewPlugin()); err != nil { + var cstore carstore.CarStore + scyllaAddrs := cctx.StringSlice("scylla-carstore") + sqliteStore := cctx.Bool("ex-sqlite-carstore") + if len(scyllaAddrs) != 0 { + slog.Info("starting scylla carstore", "addrs", scyllaAddrs) + cstore, err = carstore.NewScyllaStore(scyllaAddrs, "cs") + } else if sqliteStore { + slog.Info("starting sqlite carstore", "dir", csdir) + cstore, err = carstore.NewSqliteStore(csdir) + } else if cctx.Bool("non-archival") { + csdburl := cctx.String("carstore-db-url") + slog.Info("setting up non-archival carstore database", "url", csdburl) + csdb, err := cliutil.SetupDatabase(csdburl, cctx.Int("max-carstore-connections")) + if err != nil { + return err + } + if cctx.Bool("db-tracing") { + if err := csdb.Use(tracing.NewPlugin()); err != nil { return err } } - - cardb, err := cliutil.SetupDatabase(cctx.String("carstoredb")) + cs, err := carstore.NewNonArchivalCarstore(csdb) if err != nil { return err } - - csdir := cctx.String("carstore") - cstore, err := carstore.NewCarStore(cardb, csdir) + cstore = cs + } else { + // make standard FileCarStore + csdburl := cctx.String("carstore-db-url") + slog.Info("setting up carstore database", "url", csdburl) + csdb, err := cliutil.SetupDatabase(csdburl, cctx.Int("max-carstore-connections")) if err != nil { return err } + if cctx.Bool("db-tracing") { + if err := csdb.Use(tracing.NewPlugin()); err != nil { + return err + } + } + csdirs := []string{csdir} + if paramDirs := cctx.StringSlice("carstore-shard-dirs"); len(paramDirs) > 0 { + csdirs = paramDirs + } - repoman := repomgr.NewRepoManager(db, cstore) + for _, csd := range csdirs { + if err := os.MkdirAll(filepath.Dir(csd), os.ModePerm); err != nil { + return err + } + } + cstore, err = carstore.NewCarStore(csdb, csdirs) + } + + if err != nil { + return err + } + + // DID RESOLUTION + // 1. the outside world, PLCSerever or Web + // 2. (maybe memcached) + // 3. in-process cache + var cachedidr did.Resolver + { + mr := did.NewMultiResolver() + + didr := &plc.PLCServer{Host: cctx.String("plc-host")} + mr.AddHandler("plc", didr) + + webr := did.WebResolver{} + if cctx.Bool("crawl-insecure-ws") { + webr.Insecure = true + } + mr.AddHandler("web", &webr) + + var prevResolver did.Resolver + memcachedServers := cctx.StringSlice("did-memcached") + if len(memcachedServers) > 0 { + prevResolver = plc.NewMemcachedDidResolver(mr, time.Hour*24, memcachedServers) + } else { + prevResolver = mr + } + + cachedidr = plc.NewCachingDidResolver(prevResolver, time.Hour*24, cctx.Int("did-cache-size")) + } - evtman := events.NewEventManager() + kmgr := indexer.NewKeyManager(cachedidr, nil) - go evtman.Run() + repoman := repomgr.NewRepoManager(cstore, kmgr) - // not necessary to generate notifications, should probably make the - // indexer just take optional callbacks for notification stuff - notifman := notifs.NewNotificationManager(db, repoman.GetRecord) + var persister events.EventPersistence - didr := &api.PLCServer{Host: cctx.String("plc")} + if dpd := cctx.String("disk-persister-dir"); dpd != "" { + slog.Info("setting up disk persister") - ix, err := indexer.NewIndexer(db, notifman, evtman, didr) + pOpts := diskpersist.DefaultDiskPersistOptions() + pOpts.Retention = cctx.Duration("event-playback-ttl") + dp, err := diskpersist.NewDiskPersistence(dpd, "", db, pOpts) if err != nil { - return err + return fmt.Errorf("setting up disk persister: %w", err) + } + persister = dp + } else { + dbp, err := dbpersist.NewDbPersistence(db, cstore, nil) + if err != nil { + return fmt.Errorf("setting up db event persistence: %w", err) + } + persister = dbp + } + + evtman := events.NewEventManager(persister) + + rf := indexer.NewRepoFetcher(db, repoman, cctx.Int("max-fetch-concurrency")) + + ix, err := indexer.NewIndexer(db, evtman, cachedidr, rf, true) + if err != nil { + return err + } + defer ix.Shutdown() + + rlskip := cctx.String("bsky-social-rate-limit-skip") + ix.ApplyPDSClientSettings = func(c *xrpc.Client) { + if c.Client == nil { + c.Client = util.RobustHTTPClient() } + if strings.HasSuffix(c.Host, ".bsky.network") { + c.Client.Timeout = time.Minute * 30 + if rlskip != "" { + c.Headers = map[string]string{ + "x-ratelimit-bypass": rlskip, + } + } + } else { + // Generic PDS timeout + c.Client.Timeout = time.Minute * 1 + } + } + rf.ApplyPDSClientSettings = ix.ApplyPDSClientSettings - repoman.SetEventHandler(func(ctx context.Context, evt *repomgr.RepoEvent) { - if err := ix.HandleRepoEvent(ctx, evt); err != nil { - log.Errorw("failed to handle repo event", "err", err) + repoman.SetEventHandler(func(ctx context.Context, evt *repomgr.RepoEvent) { + if err := ix.HandleRepoEvent(ctx, evt); err != nil { + slog.Error("failed to handle repo event", "err", err) + } + }, false) + + prodHR, err := handles.NewProdHandleResolver(100_000, cctx.String("resolve-address"), cctx.Bool("force-dns-udp")) + if err != nil { + return fmt.Errorf("failed to set up handle resolver: %w", err) + } + if rlskip != "" { + prodHR.ReqMod = func(req *http.Request, host string) error { + if strings.HasSuffix(host, ".bsky.social") { + req.Header.Set("x-ratelimit-bypass", rlskip) } - }) + return nil + } + } - bgs := bgs.NewBGS(db, ix, repoman, evtman, didr) + var hr handles.HandleResolver = prodHR + if cctx.StringSlice("handle-resolver-hosts") != nil { + hr = &handles.TestHandleResolver{ + TrialHosts: cctx.StringSlice("handle-resolver-hosts"), + } + } - return bgs.Start(":2470") + slog.Info("constructing bgs") + bgsConfig := libbgs.DefaultBGSConfig() + bgsConfig.SSL = !cctx.Bool("crawl-insecure-ws") + bgsConfig.CompactInterval = cctx.Duration("compact-interval") + bgsConfig.ConcurrencyPerPDS = cctx.Int64("concurrency-per-pds") + bgsConfig.MaxQueuePerPDS = cctx.Int64("max-queue-per-pds") + bgsConfig.DefaultRepoLimit = cctx.Int64("default-repo-limit") + bgsConfig.NumCompactionWorkers = cctx.Int("num-compaction-workers") + nextCrawlers := cctx.StringSlice("next-crawler") + if len(nextCrawlers) != 0 { + nextCrawlerUrls := make([]*url.URL, len(nextCrawlers)) + for i, tu := range nextCrawlers { + var err error + nextCrawlerUrls[i], err = url.Parse(tu) + if err != nil { + return fmt.Errorf("failed to parse next-crawler url: %w", err) + } + slog.Info("configuring relay for requestCrawl", "host", nextCrawlerUrls[i]) + } + bgsConfig.NextCrawlers = nextCrawlerUrls + } + bgs, err := libbgs.NewBGS(db, ix, repoman, evtman, cachedidr, rf, hr, bgsConfig) + if err != nil { + return err + } + + if tok := cctx.String("admin-key"); tok != "" { + if err := bgs.CreateAdminToken(tok); err != nil { + return fmt.Errorf("failed to set up admin token: %w", err) + } + } + + // set up metrics endpoint + go func() { + if err := bgs.StartMetrics(cctx.String("metrics-listen")); err != nil { + log.Error("failed to start metrics endpoint", "err", err) + os.Exit(1) + } + }() + + bgsErr := make(chan error, 1) + + go func() { + err := bgs.Start(cctx.String("api-listen")) + bgsErr <- err + }() + + slog.Info("startup complete") + select { + case <-signals: + log.Info("received shutdown signal") + errs := bgs.Shutdown() + for err := range errs { + slog.Error("error during BGS shutdown", "err", err) + } + case err := <-bgsErr: + if err != nil { + slog.Error("error during BGS startup", "err", err) + } + log.Info("shutting down") + errs := bgs.Shutdown() + for err := range errs { + slog.Error("error during BGS shutdown", "err", err) + } } - app.RunAndExitOnError() + log.Info("shutdown complete") + + return nil } diff --git a/cmd/bigsky/resync_pdses.py b/cmd/bigsky/resync_pdses.py new file mode 100644 index 000000000..710aeb7d9 --- /dev/null +++ b/cmd/bigsky/resync_pdses.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# +# pip install requests +# +# python3 resync_pdses.py --admin-key hunter2 --url http://myrelay:2470 host_per_line.txt + +import json +import sys +import urllib.parse + +import requests + + +# pds limits for POST /admin/pds/changeLimits +# {"host":"", "per_second": int, "per_hour": int, "per_day": int, "crawl_rate": int, "repo_limit": int} + +limitsKeys = ('per_second', 'per_hour', 'per_day', 'crawl_rate', 'repo_limit') + +def checkLimits(limits): + for k in limits.keys(): + if k not in limitsKeys: + raise Exception(f"unknown pds rate limits key {k!r}") + return True + +class relay: + def __init__(self, rooturl, headers=None, session=None): + "rooturl string, headers dict or None, session requests.Session() or None" + self.rooturl = rooturl + self.headers = headers or dict() + self.session = session or requests.Session() + + def resync(self, host): + "host string" + url = urllib.parse.urljoin(self.rooturl, '/admin/pds/resync') + response = self.session.post(url, params={"host": host}, headers=self.headers, data='') + if response.status_code != 200: + sys.stderr.write(f"{url}?host={host} : ({response.status_code}) ({response.text!r})\n") + else: + sys.stderr.write(f"{url}?host={host} : OK\n") + + def crawlAndSetLimits(self, host, limits): + "host string, limits dict" + pheaders = dict(self.headers) + pheaders['Content-Type'] = 'application/json' + url = urllib.parse.urljoin(self.rooturl, '/admin/pds/requestCrawl') + response = self.session.post(url, headers=pheaders, data=json.dumps({"hostname": host})) + if response.status_code != 200: + sys.stderr.write(f"{url} {host} : {response.status_code} {response.text!r}\n") + return + if limits is None: + sys.stderr.write(f"requestCrawl {host} OK\n") + url = urllib.parse.urljoin(self.rooturl, '/admin/pds/changeLimits') + plimits = dict(limits) + plimits["host"] = host + response = self.session.post(url, headers=pheaders, data=json.dumps(plimits)) + if response.status_code != 200: + sys.stderr.write(f"{url} {host} : {response.status_code} {response.text!r}\n") + return + sys.stderr.write(f"requestCrawl + changeLimits {host} OK\n") + +def main(): + import argparse + ap = argparse.ArgumentParser() + ap.add_argument('input', default='-', help='host per line text file to read, - for stdin') + ap.add_argument('--admin-key', default=None, help='relay auth bearer token', required=True) + ap.add_argument('--url', default=None, help='base url to POST /admin/pds/resync', required=True) + ap.add_argument('--resync', default=False, action='store_true', help='resync selected PDSes') + ap.add_argument('--limits', default=None, help='json pds rate limits') + ap.add_argument('--crawl', default=False, action='store_true', help='crawl & set limits') + args = ap.parse_args() + + headers = {'Authorization': 'Bearer ' + args.admin_key} + + relaySession = relay(args.url, headers) + + #url = urllib.parse.urljoin(args.url, '/admin/pds/resync') + + #sess = requests.Session() + if args.crawl and args.resync: + sys.stderr.write("should only specify one of --resync --crawl") + sys.exit(1) + if (not args.crawl) and (not args.resync): + sys.stderr.write("should specify one of --resync --crawl") + sys.exit(1) + + limits = None + if args.limits: + limits = json.loads(args.limits) + checkLimits(limits) + + if args.input == '-': + fin = sys.stdin + else: + fin = open(args.input, 'rt') + for line in fin: + if not line: + continue + line = line.strip() + if not line: + continue + if line[0] == '#': + continue + host = line + if args.crawl: + relaySession.crawlAndSetLimits(host, limits) + elif args.resync: + relaySession.resync(host) + # response = sess.post(url, params={"host": line}, headers=headers) + # if response.status_code != 200: + # sys.stderr.write(f"{url}?host={line} : ({response.status_code}) ({response.text!r})\n") + # else: + # sys.stderr.write(f"{url}?host={line} : OK\n") + +if __name__ == '__main__': + main() diff --git a/cmd/bigsky/sync_pds.sh b/cmd/bigsky/sync_pds.sh new file mode 100755 index 000000000..05020bf33 --- /dev/null +++ b/cmd/bigsky/sync_pds.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -e # fail on error +set -u # fail if variable not set in substitution +set -o pipefail # fail if part of a '|' command fails + +if test -z "${RELAY_ADMIN_KEY}"; then + echo "RELAY_ADMIN_KEY secret is not defined" + exit -1 +fi + +if test -z "${RELAY_HOST}"; then + echo "RELAY_HOST config not defined" + exit -1 +fi + +if test -z "$1"; then + echo "expected PDS hostname as an argument" + exit -1 +fi + +echo "POST resync $1" +http --ignore-stdin post https://${RELAY_HOST}/admin/pds/resync Authorization:"Bearer ${RELAY_ADMIN_KEY}" \ + host==$1 diff --git a/cmd/bigsky/sync_status_pds.sh b/cmd/bigsky/sync_status_pds.sh new file mode 100755 index 000000000..bb1aa0c9f --- /dev/null +++ b/cmd/bigsky/sync_status_pds.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -e # fail on error +set -u # fail if variable not set in substitution +set -o pipefail # fail if part of a '|' command fails + +if test -z "${RELAY_ADMIN_KEY}"; then + echo "RELAY_ADMIN_KEY secret is not defined" + exit -1 +fi + +if test -z "${RELAY_HOST}"; then + echo "RELAY_HOST config not defined" + exit -1 +fi + +if test -z "$1"; then + echo "expected PDS hostname as an argument" + exit -1 +fi + +echo "GET resync $1" +http --ignore-stdin --pretty all get https://${RELAY_HOST}/admin/pds/resync Authorization:"Bearer ${RELAY_ADMIN_KEY}" \ + host==$1 diff --git a/cmd/bluepages/Dockerfile b/cmd/bluepages/Dockerfile new file mode 100644 index 000000000..ff0ac32ec --- /dev/null +++ b/cmd/bluepages/Dockerfile @@ -0,0 +1,37 @@ +# Run this dockerfile from the top level of the indigo git repository like: +# +# podman build -f ./cmd/bluepages/Dockerfile -t bluepages . + +### Compile stage +FROM golang:1.25-alpine3.22 AS build-env +RUN apk add --no-cache build-base make git + +ADD . /dockerbuild +WORKDIR /dockerbuild + +# timezone data for alpine builds +ENV GOEXPERIMENT=loopvar +RUN GIT_VERSION=$(git describe --tags --long --always) && \ + go build -tags timetzdata -o /bluepages ./cmd/bluepages + +### Run stage +FROM alpine:3.22 + +RUN apk add --no-cache --update dumb-init ca-certificates runit + +WORKDIR / +RUN mkdir -p data/bluepages +COPY --from=build-env /bluepages / + +# small things to make golang binaries work well under alpine +ENV GODEBUG=netdns=go +ENV TZ=Etc/UTC + +EXPOSE 6600 + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] +CMD ["/bluepages", "serve"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="atproto identity directory (bluepages)" +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" diff --git a/cmd/bluepages/README.md b/cmd/bluepages/README.md new file mode 100644 index 000000000..848ff8283 --- /dev/null +++ b/cmd/bluepages/README.md @@ -0,0 +1,17 @@ + +bluepages: an atproto identity directory +======================================== + +This is a simple API server which caches atproto handle and DID resolution responses. It is useful when you have a bunch of services that do identity resolution, and you don't want duplicated caches. + +Available commands, flags, and config are documented in the usage (`--help`). + +Current features and design decisions: + +- all caches stored in Redis +- will consume from the firehose (but doesn't yet) +- Lexicon API endpoints: + - `GET com.atproto.identity.resolveHandle` + - `GET com.atproto.identity.resolveDid` + - `GET com.atproto.identity.resolveIdentity` + - `POST com.atproto.identity.refreshIdentity` (admin auth) diff --git a/cmd/bluepages/firehose.go b/cmd/bluepages/firehose.go new file mode 100644 index 000000000..b6b59abe3 --- /dev/null +++ b/cmd/bluepages/firehose.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/url" + "sync/atomic" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/schedulers/parallel" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/gorilla/websocket" + "github.com/redis/go-redis/v9" +) + +var firehoseCursorKey = "bluepages/firehoseSeq" + +func (srv *Server) RunFirehoseConsumer(ctx context.Context, host string, parallelism int) error { + + cur, err := srv.ReadLastCursor(ctx) + if err != nil { + return err + } + + dialer := websocket.DefaultDialer + u, err := url.Parse(host) + if err != nil { + return fmt.Errorf("invalid Host URI: %w", err) + } + u.Path = "xrpc/com.atproto.sync.subscribeRepos" + if cur != 0 { + u.RawQuery = fmt.Sprintf("cursor=%d", cur) + } + srv.logger.Info("subscribing to repo event stream", "upstream", host, "cursor", cur) + con, _, err := dialer.Dial(u.String(), http.Header{ + "User-Agent": []string{fmt.Sprintf("bluepages/%s", versioninfo.Short())}, + }) + if err != nil { + return fmt.Errorf("subscribing to firehose failed (dialing): %w", err) + } + + rsc := &events.RepoStreamCallbacks{ + RepoIdentity: func(evt *comatproto.SyncSubscribeRepos_Identity) error { + atomic.StoreInt64(&srv.lastSeq, evt.Seq) + ctx := context.Background() + srv.logger.Info("flushing cache due to #identity firehose event", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) + + did, err := syntax.ParseDID(evt.Did) + if err != nil { + srv.logger.Warn("invalid DID in #identity event", "did", evt.Did, "seq", evt.Seq, "err", err) + return nil + } + if err := srv.dir.PurgeDID(ctx, did); err != nil { + srv.logger.Error("failed to purge DID from cache", "did", evt.Did, "seq", evt.Seq, "err", err) + return nil + } + if evt.Handle == nil { + return nil + } + handle, err := syntax.ParseHandle(*evt.Handle) + if err != nil { + srv.logger.Warn("invalid handle in #identity event", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) + return nil + } + if err := srv.dir.PurgeHandle(ctx, handle); err != nil { + srv.logger.Error("failed to purge handle from cache", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) + return nil + } + return nil + }, + } + + var scheduler events.Scheduler + // use a fixed-parallelism scheduler if configured + scheduler = parallel.NewScheduler( + parallelism, + 1000, + host, + rsc.EventHandler, + ) + srv.logger.Info("bluepages firehose scheduler configured", "scheduler", "parallel", "initial", parallelism) + + return events.HandleRepoStream(ctx, con, scheduler, srv.logger) +} + +func (srv *Server) ReadLastCursor(ctx context.Context) (int64, error) { + // if redis isn't configured, just skip + if srv.redisClient == nil { + srv.logger.Info("redis not configured, skipping cursor read") + return 0, nil + } + + val, err := srv.redisClient.Get(ctx, firehoseCursorKey).Int64() + if err == redis.Nil { + srv.logger.Info("no pre-existing cursor in redis") + return 0, nil + } else if err != nil { + return 0, err + } + srv.logger.Info("successfully found prior subscription cursor seq in redis", "seq", val) + return val, nil +} + +func (srv *Server) PersistCursor(ctx context.Context) error { + // if redis isn't configured, just skip + if srv.redisClient == nil { + return nil + } + lastSeq := atomic.LoadInt64(&srv.lastSeq) + if lastSeq <= 0 { + return nil + } + err := srv.redisClient.Set(ctx, firehoseCursorKey, lastSeq, 14*24*time.Hour).Err() + return err +} + +// this method runs in a loop, persisting the current cursor state every 5 seconds +func (srv *Server) RunPersistCursor(ctx context.Context) error { + + // if redis isn't configured, just skip + if srv.redisClient == nil { + return nil + } + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + lastSeq := atomic.LoadInt64(&srv.lastSeq) + if lastSeq >= 1 { + srv.logger.Info("persisting final cursor seq value", "seq", lastSeq) + err := srv.PersistCursor(ctx) + if err != nil { + srv.logger.Error("failed to persist cursor", "err", err, "seq", lastSeq) + } + } + return nil + case <-ticker.C: + lastSeq := atomic.LoadInt64(&srv.lastSeq) + if lastSeq >= 1 { + err := srv.PersistCursor(ctx) + if err != nil { + srv.logger.Error("failed to persist cursor", "err", err, "seq", lastSeq) + } + } + } + } +} diff --git a/cmd/bluepages/handlers.go b/cmd/bluepages/handlers.go new file mode 100644 index 000000000..7d9302d42 --- /dev/null +++ b/cmd/bluepages/handlers.go @@ -0,0 +1,261 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/labstack/echo/v4" +) + +// GET /xrpc/com.atproto.identity.resolveHandle +func (srv *Server) ResolveHandle(c echo.Context) error { + ctx := c.Request().Context() + + hdl, err := syntax.ParseHandle(c.QueryParam("handle")) + if err != nil { + return c.JSON(400, GenericError{ + Error: "InvalidHandleSyntax", + Message: err.Error(), + }) + } + + did, err := srv.dir.ResolveHandle(ctx, hdl) + if err != nil && errors.Is(err, identity.ErrHandleNotFound) { + return c.JSON(404, GenericError{ + Error: "HandleNotFound", + Message: err.Error(), + }) + } else if err != nil { + return c.JSON(500, GenericError{ + Error: "InternalError", + Message: err.Error(), + }) + } + return c.JSON(200, comatproto.IdentityResolveHandle_Output{ + Did: did.String(), + }) +} + +// GET /xrpc/com.atproto.identity.resolveDid +func (srv *Server) ResolveDid(c echo.Context) error { + ctx := c.Request().Context() + + did, err := syntax.ParseDID(c.QueryParam("did")) + if err != nil { + return c.JSON(400, GenericError{ + Error: "InvalidDidSyntax", + Message: err.Error(), + }) + } + + rawDoc, err := srv.dir.ResolveDIDRaw(ctx, did) + if err != nil && errors.Is(err, identity.ErrDIDNotFound) { + return c.JSON(404, GenericError{ + Error: "DidNotFound", + Message: err.Error(), + }) + } else if err != nil { + return c.JSON(500, GenericError{ + Error: "InternalError", + Message: err.Error(), + }) + } + return c.JSON(200, comatproto.IdentityResolveDid_Output{ + DidDoc: rawDoc, + }) +} + +// helper for resolveIdentity +func (srv *Server) resolveIdentityFromHandle(c echo.Context, handle syntax.Handle) error { + ctx := c.Request().Context() + + handle = handle.Normalize() + + did, err := srv.dir.ResolveHandle(ctx, handle) + if err != nil && errors.Is(err, identity.ErrHandleNotFound) { + return c.JSON(404, GenericError{ + Error: "HandleNotFound", + Message: err.Error(), + }) + } else if err != nil { + srv.logger.Warn("failed handle resolution", "err", err, "handle", handle) + return c.JSON(502, GenericError{ + Error: "HandleResolutionFailed", + Message: err.Error(), + }) + } + + rawDoc, err := srv.dir.ResolveDIDRaw(ctx, did) + if err != nil && errors.Is(err, identity.ErrDIDNotFound) { + return c.JSON(404, GenericError{ + Error: "DidNotFound", + Message: err.Error(), + }) + } else if err != nil { + return c.JSON(502, GenericError{ + Error: "DIDResolutionFailed", + Message: err.Error(), + }) + } + + var doc identity.DIDDocument + if err := json.Unmarshal(rawDoc, &doc); err != nil { + return c.JSON(400, GenericError{ + Error: "InvalidDidDocument", + Message: err.Error(), + }) + } + + ident := identity.ParseIdentity(&doc) + // NOTE: 'handle' was resolved above, and 'DeclaredHandle()' returns a normalized handle + declHandle, err := ident.DeclaredHandle() + if err != nil || declHandle != handle { + return c.JSON(400, GenericError{ + Error: "HandleMismatch", + Message: err.Error(), + }) + } + + return c.JSON(200, comatproto.IdentityDefs_IdentityInfo{ + Did: ident.DID.String(), + Handle: handle.String(), + DidDoc: rawDoc, + }) +} + +// helper for resolveIdentity +func (srv *Server) resolveIdentityFromDID(c echo.Context, did syntax.DID) error { + ctx := c.Request().Context() + + rawDoc, err := srv.dir.ResolveDIDRaw(ctx, did) + if err != nil && errors.Is(err, identity.ErrDIDNotFound) { + return c.JSON(404, GenericError{ + Error: "DidNotFound", + Message: err.Error(), + }) + } else if err != nil { + return c.JSON(502, GenericError{ + Error: "DIDResolutionFailed", + Message: err.Error(), + }) + } + + var doc identity.DIDDocument + if err := json.Unmarshal(rawDoc, &doc); err != nil { + return c.JSON(400, GenericError{ + Error: "InvalidDidDocument", + Message: err.Error(), + }) + } + + ident := identity.ParseIdentity(&doc) + handle, err := ident.DeclaredHandle() + if err != nil { + // no handle declared, or invalid syntax + handle = syntax.HandleInvalid + } + + checkDID, err := srv.dir.ResolveHandle(ctx, handle) + if err != nil || checkDID != did { + handle = syntax.HandleInvalid + } + + return c.JSON(200, comatproto.IdentityDefs_IdentityInfo{ + Did: ident.DID.String(), + Handle: handle.String(), + DidDoc: rawDoc, + }) +} + +// GET /xrpc/com.atproto.identity.resolveIdentity +func (srv *Server) ResolveIdentity(c echo.Context) error { + // we partially re-implement the "Lookup()" logic here, but returning the full DID document, not `identity.Identity` + atid, err := syntax.ParseAtIdentifier(c.QueryParam("identifier")) + if err != nil { + return c.JSON(400, GenericError{ + Error: "InvalidIdentifierSyntax", + Message: err.Error(), + }) + } + + handle, err := atid.AsHandle() + if nil == err { + return srv.resolveIdentityFromHandle(c, handle) + } + did, err := atid.AsDID() + if nil == err { + return srv.resolveIdentityFromDID(c, did) + } + return fmt.Errorf("unreachable code path") +} + +// POST /xrpc/com.atproto.identity.refreshIdentity +func (srv *Server) RefreshIdentity(c echo.Context) error { + ctx := c.Request().Context() + + var body comatproto.IdentityRefreshIdentity_Input + if err := c.Bind(&body); err != nil { + return c.JSON(400, GenericError{ + Error: "InvalidRequestBody", + Message: err.Error(), + }) + } + + atid, err := syntax.ParseAtIdentifier(body.Identifier) + if err != nil { + return c.JSON(400, GenericError{ + Error: "InvalidIdentifierSyntax", + Message: err.Error(), + }) + } + + did, err := atid.AsDID() + if nil == err { + if err := srv.dir.PurgeDID(ctx, did); err != nil { + return err + } + return srv.resolveIdentityFromDID(c, did) + } + handle, err := atid.AsHandle() + if nil == err { + if err := srv.dir.PurgeHandle(ctx, handle); err != nil { + return err + } + return srv.resolveIdentityFromHandle(c, handle) + } + + return fmt.Errorf("unreachable code path") +} + +type GenericStatus struct { + Daemon string `json:"daemon"` + Status string `json:"status"` + Message string `json:"msg,omitempty"` +} + +func (s *Server) HandleHealthCheck(c echo.Context) error { + return c.JSON(200, GenericStatus{Status: "ok", Daemon: "bluepages"}) +} + +func (srv *Server) WebHome(c echo.Context) error { + return c.String(200, ` +eeeee e e e eeee eeeee eeeee eeeee eeee eeeee +8 8 8 8 8 8 8 8 8 8 8 8 8 8 " +8eee8e 8e 8e 8 8eee 8eee8 8eee8 8e 8eee 8eeee +88 8 88 88 8 88 88 88 8 88 "8 88 88 +88eee8 88eee 88ee8 88ee 88 88 8 88ee8 88ee 8ee88 + +This is an AT Protocol Identity Service + +Most API routes are under /xrpc/ + + Code: https://github.com/bluesky-social/indigo/tree/main/cmd/bluepages + Protocol: https://atproto.com + `) + +} diff --git a/cmd/bluepages/main.go b/cmd/bluepages/main.go new file mode 100644 index 000000000..0676c5c69 --- /dev/null +++ b/cmd/bluepages/main.go @@ -0,0 +1,336 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "os" + "runtime" + "strings" + + _ "github.com/joho/godotenv/autoload" + _ "net/http/pprof" + + "github.com/bluesky-social/indigo/atproto/identity/apidir" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/urfave/cli/v3" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("exiting", "err", err) + os.Exit(-1) + } +} + +func run(args []string) error { + + app := cli.Command{ + Name: "bluepages", + Usage: "atproto identity directory", + Version: versioninfo.Short(), + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "atp-relay-host", + Usage: "hostname and port of Relay to subscribe to", + Value: "wss://bsky.network", + Sources: cli.EnvVars("ATP_RELAY_HOST", "ATP_BGS_HOST"), + }, + &cli.StringFlag{ + Name: "atp-plc-host", + Usage: "method, hostname, and port of PLC registry", + Value: "https://plc.directory", + Sources: cli.EnvVars("ATP_PLC_HOST"), + }, + &cli.IntFlag{ + Name: "plc-rate-limit", + Usage: "max number of requests per second to PLC registry", + Value: 300, + Sources: cli.EnvVars("BLUEPAGES_PLC_RATE_LIMIT"), + }, + &cli.StringFlag{ + Name: "redis-url", + Usage: "redis connection URL: redis://:@:6379/", + Value: "redis://localhost:6379/0", + Sources: cli.EnvVars("BLUEPAGES_REDIS_URL"), + }, + &cli.StringFlag{ + Name: "log-level", + Usage: "log verbosity level (eg: warn, info, debug)", + Sources: cli.EnvVars("BLUEPAGES_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"), + }, + }, + Commands: []*cli.Command{ + &cli.Command{ + Name: "serve", + Usage: "run the bluepages API daemon", + Action: runServeCmd, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "bind", + Usage: "Specify the local IP/port to bind to", + Required: false, + Value: ":6600", + Sources: cli.EnvVars("BLUEPAGES_BIND"), + }, + &cli.StringFlag{ + Name: "metrics-listen", + Usage: "IP or address, and port, to listen on for metrics APIs", + Value: ":3989", + Sources: cli.EnvVars("BLUEPAGES_METRICS_LISTEN"), + }, + &cli.BoolFlag{ + Name: "disable-firehose-consumer", + Usage: "don't consume #identity events from firehose", + Sources: cli.EnvVars("BLUEPAGES_DISABLE_FIREHOSE_CONSUMER"), + }, + &cli.BoolFlag{ + Name: "disable-refresh", + Usage: "disable the refreshIdentity API endpoint", + Sources: cli.EnvVars("BLUEPAGES_DISABLE_REFRESH"), + }, + &cli.IntFlag{ + Name: "firehose-parallelism", + Usage: "number of concurrent firehose workers", + Value: 4, + Sources: cli.EnvVars("BLUEPAGES_FIREHOSE_PARALLELISM"), + }, + }, + }, + &cli.Command{ + Name: "resolve-handle", + ArgsUsage: ``, + Usage: "query service for handle resoltion", + Action: runResolveHandleCmd, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Usage: "bluepages server to send request to", + Value: "http://localhost:6600", + Sources: cli.EnvVars("BLUEPAGES_HOST"), + }, + }, + }, + &cli.Command{ + Name: "resolve-did", + ArgsUsage: ``, + Usage: "query service for DID document resoltion", + Action: runResolveDIDCmd, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Usage: "bluepages server to send request to", + Value: "http://localhost:6600", + Sources: cli.EnvVars("BLUEPAGES_HOST"), + }, + }, + }, + &cli.Command{ + Name: "lookup", + ArgsUsage: ``, + Usage: "query service for identity resoltion", + Action: runLookupCmd, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Usage: "bluepages server to send request to", + Value: "http://localhost:6600", + Sources: cli.EnvVars("BLUEPAGES_HOST"), + }, + }, + }, + &cli.Command{ + Name: "refresh", + ArgsUsage: ``, + Usage: "ask service to refresh identity", + Action: runRefreshCmd, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Usage: "bluepages server to send request to", + Value: "http://localhost:6600", + Sources: cli.EnvVars("BLUEPAGES_HOST"), + }, + }, + }, + }, + } + + return app.Run(context.Background(), args) +} + +func configLogger(cmd *cli.Command, writer io.Writer) *slog.Logger { + var level slog.Level + switch strings.ToLower(cmd.String("log-level")) { + case "error": + level = slog.LevelError + case "warn": + level = slog.LevelWarn + case "info": + level = slog.LevelInfo + case "debug": + level = slog.LevelDebug + default: + level = slog.LevelInfo + } + logger := slog.New(slog.NewJSONHandler(writer, &slog.HandlerOptions{ + Level: level, + })) + slog.SetDefault(logger) + return logger +} + +func configClient(cmd *cli.Command) apidir.APIDirectory { + return apidir.NewAPIDirectory(cmd.String("host")) +} + +func runServeCmd(ctx context.Context, cmd *cli.Command) error { + logger := configLogger(cmd, os.Stdout) + + srv, err := NewServer( + Config{ + Logger: logger, + Bind: cmd.String("bind"), + RedisURL: cmd.String("redis-url"), + PLCHost: cmd.String("atp-plc-host"), + PLCRateLimit: cmd.Int("plc-rate-limit"), + DisableRefresh: cmd.Bool("disable-refresh"), + }, + ) + if err != nil { + return fmt.Errorf("failed to construct server: %v", err) + } + + if !cmd.Bool("disable-firehose-consumer") { + go func() { + firehoseHost := cmd.String("atp-relay-host") + firehoseParallelism := cmd.Int("firehose-parallelism") + if err := srv.RunFirehoseConsumer(ctx, firehoseHost, firehoseParallelism); err != nil { + slog.Error("firehose consumer thread failed", "err", err) + // NOTE: not crashing or halting process here + } + }() + go func() { + if err := srv.RunPersistCursor(ctx); err != nil { + slog.Error("firehose persist thread failed", "err", err) + // NOTE: not crashing or halting process here + } + }() + } + + // prometheus HTTP endpoint: /metrics + go func() { + // TODO: what is this tuning for? just cargo-culted it + runtime.SetBlockProfileRate(10) + runtime.SetMutexProfileFraction(10) + if err := srv.RunMetrics(cmd.String("metrics-listen")); err != nil { + slog.Error("failed to start metrics endpoint", "error", err) + // NOTE: not crashing or halting process here + } + }() + + return srv.RunAPI() +} + +func runResolveHandleCmd(ctx context.Context, cmd *cli.Command) error { + dir := configClient(cmd) + + s := cmd.Args().First() + if s == "" { + return fmt.Errorf("need to provide identifier for resolution") + } + handle, err := syntax.ParseHandle(s) + if err != nil { + return err + } + + did, err := dir.ResolveHandle(ctx, handle) + if err != nil { + return err + } + fmt.Println(did.String()) + return nil +} + +func runResolveDIDCmd(ctx context.Context, cmd *cli.Command) error { + dir := configClient(cmd) + + s := cmd.Args().First() + if s == "" { + return fmt.Errorf("need to provide identifier for resolution") + } + did, err := syntax.ParseDID(s) + if err != nil { + return err + } + + raw, err := dir.ResolveDIDRaw(ctx, did) + if err != nil { + return err + } + b, err := json.MarshalIndent(raw, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + return nil +} + +func runLookupCmd(ctx context.Context, cmd *cli.Command) error { + dir := configClient(cmd) + + s := cmd.Args().First() + if s == "" { + return fmt.Errorf("need to provide identifier for resolution") + } + atid, err := syntax.ParseAtIdentifier(s) + if err != nil { + return err + } + + ident, err := dir.Lookup(ctx, atid) + if err != nil { + return err + } + + b, err := json.MarshalIndent(ident, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + return nil +} + +func runRefreshCmd(ctx context.Context, cmd *cli.Command) error { + dir := configClient(cmd) + + s := cmd.Args().First() + if s == "" { + return fmt.Errorf("need to provide identifier for resolution") + } + atid, err := syntax.ParseAtIdentifier(s) + if err != nil { + return err + } + + err = dir.Purge(ctx, atid) + if err != nil { + return err + } + + ident, err := dir.Lookup(ctx, atid) + if err != nil { + return err + } + + b, err := json.MarshalIndent(ident, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + return nil +} diff --git a/cmd/bluepages/metrics.go b/cmd/bluepages/metrics.go new file mode 100644 index 000000000..db2c8eaf1 --- /dev/null +++ b/cmd/bluepages/metrics.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var handleResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_bluepages_resolve_handle", + Help: "ATProto handle resolutions", +}, []string{"directory", "status"}) + +var handleResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_bluepages_resolve_handle_duration", + Help: "Time to resolve a handle", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) + +var didResolution = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "atproto_identity_bluepages_resolve_did", + Help: "ATProto DID resolutions", +}, []string{"directory", "status"}) + +var didResolutionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "atproto_identity_bluepages_resolve_did_duration", + Help: "Time to resolve a DID", + Buckets: prometheus.ExponentialBucketsRange(0.001, 2, 15), +}, []string{"directory", "status"}) diff --git a/cmd/bluepages/resolver.go b/cmd/bluepages/resolver.go new file mode 100644 index 000000000..a96eca576 --- /dev/null +++ b/cmd/bluepages/resolver.go @@ -0,0 +1,319 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + + "github.com/go-redis/cache/v9" + "github.com/redis/go-redis/v9" +) + +// This file is a fork of indigo:atproto/identity/redisdir. It stores raw DID documents, not identities, and implements `identity.Resolver`. + +// Uses redis as a cache for identity lookups. +// +// Includes an in-process LRU cache as well (provided by the redis client library), for hot key (identities). +type RedisResolver struct { + Inner identity.Resolver + ErrTTL time.Duration + HitTTL time.Duration + InvalidHandleTTL time.Duration + Logger *slog.Logger + + handleCache *cache.Cache + didCache *cache.Cache + didResolveChans sync.Map + handleResolveChans sync.Map +} + +type handleEntry struct { + Updated time.Time + // needs to be pointer type, because unmarshalling empty string would be an error + DID *syntax.DID + Err error +} + +type didEntry struct { + Updated time.Time + RawDoc json.RawMessage + Err error +} + +var _ identity.Resolver = (*RedisResolver)(nil) + +// Creates a new caching `identity.Resolver` wrapper around an existing directory, using Redis and in-process LRU for caching. +// +// `redisURL` contains all the redis connection config options. +// `hitTTL` and `errTTL` define how long successful and errored identity metadata should be cached (respectively). errTTL is expected to be shorted than hitTTL. +// `lruSize` is the size of the in-process cache, for each of the handle and identity caches. 10000 is a reasonable default. +// +// NOTE: Errors returned may be inconsistent with the base directory, or between calls. This is because cached errors are serialized/deserialized and that may break equality checks. +func NewRedisResolver(inner identity.Resolver, redisURL string, hitTTL, errTTL, invalidHandleTTL time.Duration, lruSize int) (*RedisResolver, error) { + opt, err := redis.ParseURL(redisURL) + if err != nil { + return nil, fmt.Errorf("could not configure redis identity cache: %w", err) + } + rdb := redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(context.TODO()).Result() + if err != nil { + return nil, fmt.Errorf("could not connect to redis identity cache: %w", err) + } + handleCache := cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(lruSize, hitTTL), + }) + didCache := cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(lruSize, hitTTL), + }) + return &RedisResolver{ + Inner: inner, + ErrTTL: errTTL, + HitTTL: hitTTL, + InvalidHandleTTL: invalidHandleTTL, + handleCache: handleCache, + didCache: didCache, + }, nil +} + +func (d *RedisResolver) isHandleStale(e *handleEntry) bool { + if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { + return true + } + return false +} + +func (d *RedisResolver) isDIDStale(e *didEntry) bool { + if e.Err != nil && time.Since(e.Updated) > d.ErrTTL { + return true + } + return false +} + +func (d *RedisResolver) refreshHandle(ctx context.Context, h syntax.Handle) handleEntry { + start := time.Now() + did, err := d.Inner.ResolveHandle(ctx, h) + duration := time.Since(start) + + if err != nil { + d.Logger.Info("handle resolution failed", "handle", h, "duration", duration, "err", err) + handleResolution.WithLabelValues("bluepages", "error").Inc() + handleResolutionDuration.WithLabelValues("bluepages", "error").Observe(time.Since(start).Seconds()) + } else { + handleResolution.WithLabelValues("bluepages", "success").Inc() + handleResolutionDuration.WithLabelValues("bluepages", "success").Observe(time.Since(start).Seconds()) + } + if duration.Seconds() > 5.0 { + d.Logger.Info("slow handle resolution", "handle", h, "duration", duration) + } + + he := handleEntry{ + Updated: time.Now(), + DID: &did, + Err: err, + } + err = d.handleCache.Set(&cache.Item{ + Ctx: ctx, + Key: "bluepages/handle/" + h.String(), + Value: he, + TTL: d.ErrTTL, + }) + if err != nil { + d.Logger.Error("identity cache write failed", "cache", "handle", "err", err) + } + return he +} + +func (d *RedisResolver) refreshDID(ctx context.Context, did syntax.DID) didEntry { + start := time.Now() + rawDoc, err := d.Inner.ResolveDIDRaw(ctx, did) + duration := time.Since(start) + + if err != nil { + d.Logger.Info("DID resolution failed", "did", did, "duration", duration, "err", err) + didResolution.WithLabelValues("bluepages", "error").Inc() + didResolutionDuration.WithLabelValues("bluepages", "error").Observe(time.Since(start).Seconds()) + } else { + didResolution.WithLabelValues("bluepages", "success").Inc() + didResolutionDuration.WithLabelValues("bluepages", "success").Observe(time.Since(start).Seconds()) + } + if duration.Seconds() > 5.0 { + d.Logger.Info("slow DID resolution", "did", did, "duration", duration) + } + + // persist the DID lookup error, instead of processing it immediately + entry := didEntry{ + Updated: time.Now(), + RawDoc: rawDoc, + Err: err, + } + + err = d.didCache.Set(&cache.Item{ + Ctx: ctx, + Key: "bluepages/did/" + did.String(), + Value: entry, + TTL: d.HitTTL, + }) + if err != nil { + d.Logger.Error("DID cache write failed", "cache", "did", "did", did, "err", err) + } + return entry +} + +func (d *RedisResolver) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { + if h.IsInvalidHandle() { + return "", fmt.Errorf("can not resolve handle: %w", identity.ErrInvalidHandle) + } + h = h.Normalize() + var entry handleEntry + err := d.handleCache.Get(ctx, "bluepages/handle/"+h.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + return "", fmt.Errorf("identity cache read failed: %w", err) + } + if err == nil && !d.isHandleStale(&entry) { // if no error... + handleResolution.WithLabelValues("bluepages", "cached").Inc() + if entry.Err != nil { + return "", entry.Err + } else if entry.DID != nil { + return *entry.DID, nil + } else { + return "", errors.New("code flow error in redis identity directory") + } + } + + // Coalesce multiple requests for the same Handle + res := make(chan struct{}) + val, loaded := d.handleResolveChans.LoadOrStore(h.String(), res) + if loaded { + handleResolution.WithLabelValues("bluepages", "coalesced").Inc() + // Wait for the result from the pending request + select { + case <-val.(chan struct{}): + // The result should now be in the cache + err := d.handleCache.Get(ctx, "bluepages/handle/"+h.String(), entry) + if err != nil && err != cache.ErrCacheMiss { + return "", fmt.Errorf("identity cache read failed: %w", err) + } + if err == nil && !d.isHandleStale(&entry) { // if no error... + if entry.Err != nil { + return "", entry.Err + } else if entry.DID != nil { + return *entry.DID, nil + } else { + return "", errors.New("code flow error in redis identity directory") + } + } + return "", errors.New("identity not found in cache after coalesce returned") + case <-ctx.Done(): + return "", ctx.Err() + } + } + + // Update the Handle Entry from PLC and cache the result + newEntry := d.refreshHandle(ctx, h) + + // Cleanup the coalesce map and close the results channel + d.handleResolveChans.Delete(h.String()) + // Callers waiting will now get the result from the cache + close(res) + + if newEntry.Err != nil { + return "", newEntry.Err + } + if newEntry.DID != nil { + return *newEntry.DID, nil + } + return "", errors.New("unexpected control-flow error") +} + +func (d *RedisResolver) ResolveDIDRaw(ctx context.Context, did syntax.DID) (json.RawMessage, error) { + var entry didEntry + err := d.didCache.Get(ctx, "bluepages/did/"+did.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + return nil, fmt.Errorf("DID cache read failed: %w", err) + } + if err == nil && !d.isDIDStale(&entry) { // if no error... + didResolution.WithLabelValues("bluepages", "cached").Inc() + return entry.RawDoc, entry.Err + } + + // Coalesce multiple requests for the same DID + res := make(chan struct{}) + val, loaded := d.didResolveChans.LoadOrStore(did.String(), res) + if loaded { + didResolution.WithLabelValues("bluepages", "coalesced").Inc() + // Wait for the result from the pending request + select { + case <-val.(chan struct{}): + // The result should now be in the cache + err = d.didCache.Get(ctx, "bluepages/did/"+did.String(), &entry) + if err != nil && err != cache.ErrCacheMiss { + return nil, fmt.Errorf("DID cache read failed: %w", err) + } + if err == nil && !d.isDIDStale(&entry) { // if no error... + return entry.RawDoc, entry.Err + } + return nil, errors.New("DID not found in cache after coalesce returned") + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + // Update the DID Entry and cache the result + newEntry := d.refreshDID(ctx, did) + + // Cleanup the coalesce map and close the results channel + d.didResolveChans.Delete(did.String()) + // Callers waiting will now get the result from the cache + close(res) + + if newEntry.Err != nil { + return nil, newEntry.Err + } + if newEntry.RawDoc != nil { + return newEntry.RawDoc, nil + } + return nil, errors.New("unexpected control-flow error") +} + +func (d *RedisResolver) ResolveDID(ctx context.Context, did syntax.DID) (*identity.DIDDocument, error) { + b, err := d.ResolveDIDRaw(ctx, did) + if err != nil { + return nil, err + } + + var doc identity.DIDDocument + if err := json.Unmarshal(b, &doc); err != nil { + return nil, fmt.Errorf("%w: JSON DID document parse: %w", identity.ErrDIDResolutionFailed, err) + } + if doc.DID != did { + return nil, fmt.Errorf("document ID did not match DID") + } + return &doc, nil +} + +func (d *RedisResolver) PurgeHandle(ctx context.Context, handle syntax.Handle) error { + handle = handle.Normalize() + err := d.handleCache.Delete(ctx, "bluepages/handle/"+handle.String()) + if err == cache.ErrCacheMiss { + return nil + } + return err +} + +func (d *RedisResolver) PurgeDID(ctx context.Context, did syntax.DID) error { + err := d.didCache.Delete(ctx, "bluepages/did/"+did.String()) + if err == cache.ErrCacheMiss { + return nil + } + return err +} diff --git a/cmd/bluepages/server.go b/cmd/bluepages/server.go new file mode 100644 index 000000000..b2b26d5bb --- /dev/null +++ b/cmd/bluepages/server.go @@ -0,0 +1,219 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/redis/go-redis/v9" + slogecho "github.com/samber/slog-echo" + "golang.org/x/time/rate" +) + +type Server struct { + dir *RedisResolver + echo *echo.Echo + httpd *http.Server + logger *slog.Logger + + // this redis client is used to store firehose offset + redisClient *redis.Client + + // lastSeq is the most recent event sequence number we've received and begun to handle. + // This number is periodically persisted to redis, if redis is present. + // The value is best-effort (the stream handling itself is concurrent, so event numbers may not be monotonic), + // but nonetheless, you must use atomics when updating or reading this (to avoid data races). + lastSeq int64 +} + +type Config struct { + Logger *slog.Logger + PLCHost string + PLCRateLimit int + RedisURL string + Bind string + DisableRefresh bool +} + +func NewServer(config Config) (*Server, error) { + logger := config.Logger + if logger == nil { + logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })) + } + + baseDir := identity.BaseDirectory{ + PLCURL: config.PLCHost, + HTTPClient: http.Client{ + Timeout: time.Second * 10, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + // would want this around 100ms for services doing lots of handle resolution (to reduce number of idle connections). Impacts PLC connections as well, but not too bad. + IdleConnTimeout: time.Millisecond * 100, + MaxIdleConns: 1000, + }, + }, + Resolver: net.Resolver{ + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{Timeout: time.Second * 3} + return d.DialContext(ctx, network, address) + }, + }, + PLCLimiter: rate.NewLimiter(rate.Limit(config.PLCRateLimit), 1), + TryAuthoritativeDNS: true, + SkipDNSDomainSuffixes: []string{".bsky.social", ".staging.bsky.dev"}, + // TODO: UserAgent: "bluepages", + } + + // TODO: config these timeouts + redisDir, err := NewRedisResolver(&baseDir, config.RedisURL, time.Hour*24, time.Minute*2, time.Minute*5, 50_000) + if err != nil { + return nil, err + } + redisDir.Logger = logger + + // configure redis client (for firehose consumer) + redisOpt, err := redis.ParseURL(config.RedisURL) + if err != nil { + return nil, fmt.Errorf("parsing redis URL: %v", err) + } + redisClient := redis.NewClient(redisOpt) + // check redis connection + _, err = redisClient.Ping(context.Background()).Result() + if err != nil { + return nil, fmt.Errorf("redis ping failed: %v", err) + } + + e := echo.New() + + // httpd + var ( + httpTimeout = 1 * time.Minute + httpMaxHeaderBytes = 1 * (1024 * 1024) + ) + + srv := &Server{ + echo: e, + dir: redisDir, + logger: logger, + redisClient: redisClient, + } + + srv.httpd = &http.Server{ + Handler: srv, + Addr: config.Bind, + WriteTimeout: httpTimeout, + ReadTimeout: httpTimeout, + MaxHeaderBytes: httpMaxHeaderBytes, + } + + e.HideBanner = true + e.Use(slogecho.New(logger)) + e.Use(middleware.Recover()) + e.Use(middleware.BodyLimit("4M")) + e.HTTPErrorHandler = srv.errorHandler + e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ + ContentTypeNosniff: "nosniff", + XFrameOptions: "SAMEORIGIN", + HSTSMaxAge: 31536000, // 365 days + // TODO: + // ContentSecurityPolicy + // XSSProtection + })) + + e.GET("/", srv.WebHome) + e.GET("/_health", srv.HandleHealthCheck) + e.GET("/xrpc/com.atproto.identity.resolveHandle", srv.ResolveHandle) + e.GET("/xrpc/com.atproto.identity.resolveDid", srv.ResolveDid) + e.GET("/xrpc/com.atproto.identity.resolveIdentity", srv.ResolveIdentity) + if !config.DisableRefresh { + e.POST("/xrpc/com.atproto.identity.refreshIdentity", srv.RefreshIdentity) + } + + return srv, nil +} + +func (srv *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + srv.echo.ServeHTTP(rw, req) +} + +func (srv *Server) RunAPI() error { + srv.logger.Info("starting server", "bind", srv.httpd.Addr) + go func() { + if err := srv.httpd.ListenAndServe(); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + srv.logger.Error("HTTP server shutting down unexpectedly", "err", err) + } + } + }() + + // Wait for a signal to exit. + srv.logger.Info("registering OS exit signal handler") + quit := make(chan struct{}) + exitSignals := make(chan os.Signal, 1) + signal.Notify(exitSignals, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-exitSignals + srv.logger.Info("received OS exit signal", "signal", sig) + + // Shut down the HTTP server + if err := srv.Shutdown(); err != nil { + srv.logger.Error("HTTP server shutdown error", "err", err) + } + + // Trigger the return that causes an exit. + close(quit) + }() + <-quit + srv.logger.Info("graceful shutdown complete") + return nil +} + +func (srv *Server) RunMetrics(bind string) error { + p := "/metrics" + srv.logger.Info("starting metrics endpoint", "bind", bind, "path", p) + http.Handle(p, promhttp.Handler()) + return http.ListenAndServe(bind, nil) +} + +func (srv *Server) Shutdown() error { + srv.logger.Info("shutting down") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return srv.httpd.Shutdown(ctx) +} + +type GenericError struct { + Error string `json:"error"` + Message string `json:"message"` +} + +func (srv *Server) errorHandler(err error, c echo.Context) { + code := http.StatusInternalServerError + var errorMessage string + if he, ok := err.(*echo.HTTPError); ok { + code = he.Code + errorMessage = fmt.Sprintf("%s", he.Message) + } + if code >= 500 { + srv.logger.Warn("bluepages-http-internal-error", "err", err) + } + if !c.Response().Committed { + c.JSON(code, GenericError{Error: "InternalError", Message: errorMessage}) + } +} diff --git a/cmd/collectiondir/Dockerfile b/cmd/collectiondir/Dockerfile new file mode 100644 index 000000000..82397f0dc --- /dev/null +++ b/cmd/collectiondir/Dockerfile @@ -0,0 +1,43 @@ +FROM golang:1.25-bookworm AS build-env + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC +ENV GODEBUG="netdns=go" +ENV GOOS="linux" +ENV GOARCH="amd64" +ENV CGO_ENABLED="1" + +WORKDIR /usr/src/collectiondir + +COPY . . + +RUN go mod download && \ + go mod verify + +RUN go build \ + -v \ + -trimpath \ + -tags timetzdata \ + -o /collectiondir-bin \ + ./cmd/collectiondir + +FROM debian:bookworm-slim + +ENV DEBIAN_FRONTEND="noninteractive" +ENV TZ=Etc/UTC +ENV GODEBUG="netdns=go" + +RUN apt-get update && apt-get install --yes \ + dumb-init \ + ca-certificates \ + runit + +WORKDIR /collectiondir +COPY --from=build-env /collectiondir-bin /usr/bin/collectiondir + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] +CMD ["/usr/bin/collectiondir"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="collectiondir " +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" diff --git a/cmd/collectiondir/README.md b/cmd/collectiondir/README.md new file mode 100644 index 000000000..f5595bfa9 --- /dev/null +++ b/cmd/collectiondir/README.md @@ -0,0 +1,56 @@ + +`collectiondir`: Directory of Accounts by Collection +==================================================== + +This is a small atproto microservice which maintains a directory of which accounts in the network (DIDs) have data (records) for which collections (NSIDs). + +It primarily serves the `com.atproto.sync.listReposByCollection` API endpoint: + +``` +GET /xrpc/com.atproto.sync.listReposByCollection?collection=com.atproto.sync.listReposByCollection?collection=com.atproto.lexicon.schema&limit=3 + +{ + "repos": [ + { "did": "did:plc:4sm3vprfyl55ui3yhjd7w4po" }, + { "did": "did:plc:xhkqwjmxuo65vwbwuiz53qor" }, + { "did": "did:plc:w3aonw33w3mz3mwws34x5of6" } + ], + "cursor": "QQAAAEkAAAGVgFFLb2RpZDpwbGM6dzNhb253MzN3M216M213d3MzNHg1b2Y2AA==" +} +``` + +Features and design points: + +- persists data in a local key/value database (pebble) +- consumes from the firehose to stay up to date with record creation +- can bootstrap the full network using `com.atproto.sync.listRepos` and `com.atproto.repo.describeRepo` +- single golang binary for easy deployment + + +## Analytics Endpoint + +``` +/v1/listCollections?c={}&cursor={}&limit={50<=limit<=1000} +``` + +`listCollections` returns JSON with a map of collection name to approximate number of dids implementing it. +With no `c` parameter it returns all known collections with cursor paging. +With up to 20 repeated `c` parameters it returns only those collections (no paging). +It may be the cached result of a computation, up to several minutes out of date. +```json +{"collections":{"app.bsky.feed.post": 123456789, "some collection": 42}, +"cursor":"opaque text"} +``` + + +## Database Schema + +The primary database is (collection, seen time int64 milliseconds, did) + +This allows for efficient cursor fetching of more dids for a collection. + +e.g. A new service starts consuming the firehose for events it wants in collection `com.newservice.data.thing`, +it then calls the collection directory for a list of repos which may have already created data in this collection, +and does `getRepo` calls to those repo's PDSes to get prior data. +By the time it is done paging forward through the collection directory results and getting those repos, +it will have backfilled data and new data it has collected live off the firehose. diff --git a/cmd/collectiondir/collectiondir.go b/cmd/collectiondir/collectiondir.go new file mode 100644 index 000000000..cbf533dce --- /dev/null +++ b/cmd/collectiondir/collectiondir.go @@ -0,0 +1,349 @@ +package main + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/urfave/cli/v3" +) + +func main() { + app := cli.Command{ + Name: "collectiondir", + Usage: "collection directory service", + Version: versioninfo.Short(), + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "verbose", + }, + }, + Commands: []*cli.Command{ + serveCmd, + offlineCrawlCmd, + buildCmd, + statsCmd, + exportCmd, + adminCrawlCmd, + }, + } + if err := app.Run(context.Background(), os.Args); err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + os.Exit(1) + } +} + +var statsCmd = &cli.Command{ + Name: "stats", + Usage: "read stats from a pebble db", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "pebble", + Usage: "path to pebble db", + Required: true, + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + logLevel := slog.LevelInfo + if cmd.Bool("verbose") { + logLevel = slog.LevelDebug + } + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) + slog.SetDefault(log) + pebblePath := cmd.String("pebble") + var db PebbleCollectionDirectory + db.log = log + err := db.Open(pebblePath) + if err != nil { + return err + } + defer db.Close() + + stats, err := db.GetCollectionStats() + if err != nil { + return err + } + blob, err := json.MarshalIndent(stats, "", " ") + os.Stdout.Write(blob) + os.Stdout.Write([]byte{'\n'}) + return nil + }, +} + +var exportCmd = &cli.Command{ + Name: "export", + Usage: "export a pebble db to CSV on stdout", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "pebble", + Usage: "path to pebble db", + Required: true, + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + logLevel := slog.LevelInfo + if cmd.Bool("verbose") { + logLevel = slog.LevelDebug + } + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) + slog.SetDefault(log) + pebblePath := cmd.String("pebble") + var db PebbleCollectionDirectory + db.log = log + err := db.Open(pebblePath) + if err != nil { + return err + } + defer db.Close() + + rows := make(chan CollectionDidTime, 100) + go func() { + err := db.ReadAllPrimary(ctx, rows) + if err != nil { + log.Error("db read", "path", pebblePath, "err", err) + } + }() + + writer := csv.NewWriter(os.Stdout) + defer writer.Flush() + err = writer.Write([]string{"did", "collection", "millis"}) + if err != nil { + log.Error("csv write header", "err", err) + } + var row [3]string + for rowi := range rows { + row[0] = rowi.Did + row[1] = rowi.Collection + row[2] = strconv.FormatInt(rowi.UnixMillis, 10) + err = writer.Write(row[:]) + if err != nil { + log.Error("csv write row", "err", err) + } + } + + return nil + }, +} + +var buildCmd = &cli.Command{ + Name: "build", + Usage: "collect csv into a database", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "csv", + Required: true, + }, + &cli.StringFlag{ + Name: "pebble", + Usage: "path to store pebble db", + Required: true, + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + logLevel := slog.LevelInfo + if cmd.Bool("verbose") { + logLevel = slog.LevelDebug + } + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) + slog.SetDefault(log) + pebblePath := cmd.String("pebble") + var db PebbleCollectionDirectory + db.log = log + err := db.Open(pebblePath) + if err != nil { + return err + } + defer db.Close() + csvPath := cmd.String("csv") + var fin io.Reader + if csvPath == "-" { + fin = os.Stdin + } else if strings.HasSuffix(csvPath, ".gz") { + osin, err := os.Open(csvPath) + if err != nil { + return fmt.Errorf("%s: could not open csv, %w", csvPath, err) + } + defer osin.Close() + gzin, err := gzip.NewReader(osin) + if err != nil { + return fmt.Errorf("%s: could not open csv, %w", csvPath, err) + } + defer gzin.Close() + fin = gzin + } else { + osin, err := os.Open(csvPath) + if err != nil { + return fmt.Errorf("%s: could not open csv, %w", csvPath, err) + } + defer osin.Close() + fin = osin + } + reader := csv.NewReader(fin) + rowcount := 0 + results := make(chan DidCollection, 100) + go db.SetFromResults(results) + for { + row, err := reader.Read() + if errors.Is(err, io.EOF) { + break + } + did := row[0] + collection := row[1] + results <- DidCollection{ + Did: did, + Collection: collection, + } + rowcount++ + } + close(results) + log.Debug("read csv", "rows", rowcount) + return nil + }, +} + +var adminCrawlCmd = &cli.Command{ + Name: "crawl", + Usage: "admin service to crawl one or more PDSes", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "csv", + Usage: "path to load csv from, use column 'host' or 'hostname'", + }, + &cli.StringFlag{ + Name: "list", + Usage: "path to load hostname list from, one per line", + }, + &cli.StringFlag{ + Name: "url", + Usage: "host:port of collectiondir server", + Required: true, + Sources: cli.EnvVars("COLLECTIONDIR_URL"), + }, + &cli.StringFlag{ + Name: "auth", + Usage: "Auth token for admin api", + Sources: cli.EnvVars("ADMIN_AUTH"), + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + logLevel := slog.LevelInfo + if cmd.Bool("verbose") { + logLevel = slog.LevelDebug + } + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) + slog.SetDefault(log) + + var hostList []string + + serverUrl, err := url.Parse(cmd.String("url")) + if err != nil { + var e2 error + // try to fixup a bare host:port which can confuse url.Parse + serverUrl, e2 = url.Parse("http://" + cmd.String("url")) + if e2 != nil { + return fmt.Errorf("could not parse url, %w", err) + } + } + requestCrawlUrl := serverUrl.JoinPath("/admin/pds/requestCrawl") + + if cmd.IsSet("list") { + fin, err := os.Open(cmd.String("list")) + if err != nil { + return fmt.Errorf("%s: could not open, %w", cmd.String("list"), err) + } + defer fin.Close() + bufin := bufio.NewScanner(fin) + for bufin.Scan() { + hostList = append(hostList, bufin.Text()) + } + err = bufin.Err() + if err != nil { + return fmt.Errorf("%s: error reading, %w", cmd.String("list"), err) + } + } else if cmd.IsSet("csv") { + fin, err := os.Open(cmd.String("csv")) + if err != nil { + return fmt.Errorf("%s: could not open, %w", cmd.String("csv"), err) + } + defer fin.Close() + data, err := csv.NewReader(fin).ReadAll() + if err != nil { + return fmt.Errorf("%s: could not read, %w", cmd.String("csv"), err) + } + if len(data) < 2 { + return fmt.Errorf("%s: empty CSV file", cmd.String("csv")) + } + headerRow := data[0] + hostCol := -1 + for i, v := range headerRow { + v = strings.ToLower(v) + if v == "host" || v == "hostname" { + hostCol = i + break + } + } + if hostCol < 0 { + return fmt.Errorf("%s: header missing 'host' or 'hostname'", cmd.String("csv")) + } + for _, row := range data[1:] { + hostList = append(hostList, row[hostCol]) + } + } + + if len(hostList) == 0 { + fmt.Println("no hosts") + } + + client := http.Client{Timeout: 1 * time.Second} + var headers http.Header = make(http.Header) + if cmd.IsSet("auth") { + headers.Add("Authorization", "Bearer "+cmd.String("auth")) + } + headers.Add("Content-Type", "application/json") + var response *http.Response + postReqeust := CrawlRequest{ + Hosts: hostList, + } + reqBlob, err := json.Marshal(postReqeust) + reqReader := bytes.NewReader(reqBlob) + + for try := 0; try < 3; try++ { + req, err := http.NewRequest("POST", requestCrawlUrl.String(), reqReader) + if err != nil { + return fmt.Errorf("could not create request, %w", err) + } + req.Header = headers + response, err = client.Do(req) + if err == nil && response.StatusCode == 200 { + break + } else { + log.Info("http err", "err", err, "status", response.StatusCode) + if try < 2 { + time.Sleep(time.Duration(try+1) * 2 * time.Second) + } + } + } + if err != nil { + return fmt.Errorf("POST %s err %w", requestCrawlUrl.String(), err) + } + if response.StatusCode != http.StatusOK { + return fmt.Errorf("POST %s err %s", requestCrawlUrl.String(), response.Status) + } + + return nil + }, +} diff --git a/cmd/collectiondir/crawl.go b/cmd/collectiondir/crawl.go new file mode 100644 index 000000000..f6679e64f --- /dev/null +++ b/cmd/collectiondir/crawl.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "log/slog" + "net/url" + "os" + "strings" + "sync/atomic" + + "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/urfave/cli/v3" + "golang.org/x/time/rate" +) + +type DidCollection struct { + Did string `json:"d"` + Collection string `json:"c"` +} + +func DidCollectionsToCsv(out io.Writer, sources <-chan DidCollection) { + writer := csv.NewWriter(out) + defer writer.Flush() + var row [2]string + for dc := range sources { + row[0] = dc.Did + row[1] = dc.Collection + writer.Write(row[:]) + } +} + +var offlineCrawlCmd = &cli.Command{ + Name: "offline_crawl", + Usage: "crawl a PDS to csv out", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Usage: "hostname or URL of PDS", + }, + &cli.StringFlag{ + Name: "csv-out", + Usage: "path for output or - for stdout", + }, + &cli.Float64Flag{ + Name: "qps", + Usage: "queries per second to do vs target PDS", + Value: 50, // large PDS: 500_000 repos, 10_000 seconds, ~3 hours + }, + &cli.StringFlag{ + Name: "ratelimit-header", + Usage: "secret for friend PDSes", + Sources: cli.EnvVars("BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"), + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hostname := cmd.String("host") + hosturl, err := url.Parse(hostname) + if err != nil { + hosturl = new(url.URL) + hosturl.Scheme = "https" + hosturl.Host = hostname + } + rpcClient := xrpc.Client{ + Host: hosturl.String(), + Client: util.RobustHTTPClient(), + } + if cmd.IsSet("ratelimit-header") { + rpcClient.Headers = map[string]string{ + "x-ratelimit-bypass": cmd.String("ratelimit-header"), + } + } + log.Info("will crawl", "url", rpcClient.Host) + csvOutPath := cmd.String("csv-out") + var fout io.Writer = os.Stdout + if csvOutPath != "" { + if csvOutPath == "-" { + fout = os.Stdout + } else { + fout, err = os.Create(csvOutPath) + if err != nil { + return fmt.Errorf("%s: could not open for writing: %w", csvOutPath, err) + } + } + } + qps := cmd.Float64("qps") + results := make(chan DidCollection, 100) + defer close(results) + go DidCollectionsToCsv(fout, results) + crawler := Crawler{ + Ctx: ctx, + RpcClient: &rpcClient, + QPS: qps, + Results: results, + Log: log, + } + err = crawler.CrawlPDSRepoCollections() + log.Info("done") + + return err + }, +} + +type Crawler struct { + Ctx context.Context + RpcClient *xrpc.Client + QPS float64 + Results chan<- DidCollection + Log *slog.Logger + Stats *CrawlStats +} + +type CrawlStats struct { + ReposDescribed atomic.Uint32 +} + +// CrawlPDSRepoCollections +// write results to chan +// does _not_ close chan +// (allow multiple threads of PDS queries running to one output chan, e.g. feeding into SetFromResults() ) +func (cr *Crawler) CrawlPDSRepoCollections() error { + var cursor string + limiter := rate.NewLimiter(rate.Limit(cr.QPS), 1) + for { + limiter.Wait(cr.Ctx) + repos, err := atproto.SyncListRepos(cr.Ctx, cr.RpcClient, cursor, 1000) + if err != nil { + return fmt.Errorf("%s: sync repos: %w", cr.RpcClient.Host, err) + } + pdsRepoPages.Inc() + slog.Debug("got repo list", "count", len(repos.Repos)) + for _, xr := range repos.Repos { + limiter.Wait(cr.Ctx) + desc, err := atproto.RepoDescribeRepo(cr.Ctx, cr.RpcClient, xr.Did) + if err != nil { + erst := err.Error() + if strings.Contains(erst, "RepoDeactivated") || strings.Contains(erst, "RepoTakendown") { + slog.Info("repo unavail", "host", cr.RpcClient.Host, "did", xr.Did, "err", err) + } else { + slog.Warn("repo desc", "host", cr.RpcClient.Host, "did", xr.Did, "err", err) + } + continue + } + pdsReposDescribed.Inc() + for _, collection := range desc.Collections { + cr.Results <- DidCollection{Did: xr.Did, Collection: collection} + } + if cr.Stats != nil { + cr.Stats.ReposDescribed.Add(1) + } + } + if repos.Cursor != nil { + cursor = *repos.Cursor + } else { + break + } + } + return nil +} diff --git a/cmd/collectiondir/firehose.go b/cmd/collectiondir/firehose.go new file mode 100644 index 000000000..3edf8be9e --- /dev/null +++ b/cmd/collectiondir/firehose.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "net/url" + "time" + + "github.com/bluesky-social/indigo/events" + + "github.com/gorilla/websocket" +) + +type Firehose struct { + Log *slog.Logger + + Host string + Seq int64 + + events chan<- *events.XRPCStreamEvent +} + +func (fh *Firehose) subscribeWithRedialer(ctx context.Context, fhevents chan<- *events.XRPCStreamEvent) error { + defer close(fhevents) + d := websocket.Dialer{} + + rurl, err := url.Parse(fh.Host) + if err != nil { + rurl = new(url.URL) + rurl.Host = fh.Host + rurl.Scheme = "wss" + } else { + if rurl.Scheme == fh.Host { + rurl.Scheme = "wss" + } + if rurl.Scheme == "https" || rurl.Scheme == "wss" { + rurl.Scheme = "wss" + } else if rurl.Scheme == "http" || rurl.Scheme == "ws" { + rurl.Scheme = "ws" + } else if rurl.Scheme == "" { + rurl.Scheme = "wss" + } else { + return fmt.Errorf("host unknown scheme %#v", rurl.Scheme) + } + } + //protocol := "wss" + subscribeReposUrl := rurl.JoinPath("/xrpc/com.atproto.sync.subscribeRepos") + fh.events = fhevents + + var backoff int + for { + select { + case <-ctx.Done(): + return nil + default: + } + + header := http.Header{ + "User-Agent": []string{"collectiondir"}, + } + + if fh.Seq >= 0 { + subscribeReposUrl.RawQuery = fmt.Sprintf("cursor=%d", fh.Seq) + } + url := subscribeReposUrl.String() + con, res, err := d.DialContext(ctx, url, header) + if err != nil { + fh.Log.Warn("dialing failed", "url", url, "err", err, "backoff", backoff) + time.Sleep(time.Duration(5+backoff) * time.Second) + backoff++ + + continue + } else { + backoff = 0 + } + + fh.Log.Info("event subscription response", "code", res.StatusCode) + + if err := fh.handleConnection(ctx, con); err != nil { + fh.Log.Warn("connection failed", "host", fh.Host, "err", err) + } + } +} + +func (fh *Firehose) handleConnection(ctx context.Context, con *websocket.Conn) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + return events.HandleRepoStream(ctx, con, fh, fh.Log) +} + +// AddWork is part of events.Scheduler +func (fh *Firehose) AddWork(ctx context.Context, repo string, val *events.XRPCStreamEvent) error { + tsv, ok := val.GetSequence() + if ok { + fh.Seq = tsv + } + fh.events <- val + return nil +} + +// Shutdown is part of events.Scheduler +func (fh *Firehose) Shutdown() { + // unneeded in this usage +} diff --git a/cmd/collectiondir/metrics.go b/cmd/collectiondir/metrics.go new file mode 100644 index 000000000..993e39af0 --- /dev/null +++ b/cmd/collectiondir/metrics.go @@ -0,0 +1,49 @@ +package main + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var firehoseReceivedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "collectiondir_firehose_received_total", + Help: "number of events received from upstream firehose", +}) +var firehoseCommits = promauto.NewCounter(prometheus.CounterOpts{ + Name: "collectiondir_firehose_commits", + Help: "number of #commit events received from upstream firehose", +}) +var firehoseCommitOps = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "collectiondir_firehose_commit_ops", + Help: "number of #commit events received from upstream firehose", +}, []string{"op"}) + +var firehoseDidcSet = promauto.NewCounter(prometheus.CounterOpts{ + Name: "collectiondir_firehose_didc_total", +}) + +var pebbleDup = promauto.NewCounter(prometheus.CounterOpts{ + Name: "collectiondir_pebble_dup_total", +}) + +var pebbleNew = promauto.NewCounter(prometheus.CounterOpts{ + Name: "collectiondir_pebble_new_total", +}) + +var pdsCrawledCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "collectiondir_pds_crawled_total", +}) + +var pdsRepoPages = promauto.NewCounter(prometheus.CounterOpts{ + Name: "collectiondir_pds_repo_pages", +}) + +var pdsReposDescribed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "collectiondir_pds_repos_described", +}) + +var statsCalculations = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "collectiondir_stats_calculations", + Help: "how long it takes to calculate total stats", + Buckets: prometheus.ExponentialBuckets(0.01, 2, 13), +}) diff --git a/cmd/collectiondir/pebble.go b/cmd/collectiondir/pebble.go new file mode 100644 index 000000000..2f6879cf4 --- /dev/null +++ b/cmd/collectiondir/pebble.go @@ -0,0 +1,422 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/cockroachdb/pebble" +) + +func makeCollectionInternKey(collection string) []byte { + out := make([]byte, len(collection)+1) + out[0] = 'C' + copy(out[1:], collection) + return out +} + +func parseCollectionInternKey(key []byte) string { + if key[0] != 'C' { + panic(fmt.Sprintf("collection key must start with C, got %v", key[0])) + } + return string(key[1:]) +} + +func makePrimaryPebbleRow(collectionId uint32, did string, seenMs int64) []byte { + out := make([]byte, 1+4+8+len(did)) + out[0] = 'A' + binary.BigEndian.PutUint32(out[1:], collectionId) + pos := 1 + 4 + binary.BigEndian.PutUint64(out[pos:], uint64(seenMs)) + pos += 8 + copy(out[pos:], did) + return out +} + +func parsePrimaryPebbleRow(row []byte) (collectionId uint32, did string, seenMs int64) { + if row[0] != 'A' { + panic(fmt.Sprintf("primary row key wanted A got %v", row[0])) + } + collectionId = binary.BigEndian.Uint32(row[1:5]) + seenMs = int64(binary.BigEndian.Uint64(row[5:13])) + did = string(row[13:]) + return collectionId, did, seenMs +} + +func makeByDidKey(did string, collectionId uint32) []byte { + out := make([]byte, 1+len(did)+4) + out[0] = 'D' + copy(out[1:1+len(did)], did) + pos := 1 + len(did) + binary.BigEndian.PutUint32(out[pos:], collectionId) + return out +} + +func parseByDidKey(key []byte) (did string, collectionId uint32) { + if key[0] != 'D' { + panic(fmt.Sprintf("by did key wanted D got %v", key[0])) + } + last4 := len(key) - 5 + collectionId = binary.BigEndian.Uint32(key[last4:]) + did = string(key[1 : last4+1]) + return did, collectionId +} + +// PebbleCollectionDirectory holds a DID<=>{collections} directory in pebble db. +// The primary database is (collection, seen time int64 milliseconds, did) +// Inner schema: +// C{collection} : {uint32 collectionId} +// D{did}{uint32 collectionId} : {uint64 seen ms} +// A{uint32 collectionId}{uint64 seen ms}{did} : 't' +type PebbleCollectionDirectory struct { + db *pebble.DB + + // collections can be LRU cache if it ever becomes too big + collections map[string]uint32 + collectionNames map[uint32]string // TODO: B-tree would be nice + maxCollectionId uint32 + collectionsLock sync.Mutex + + log *slog.Logger +} + +func (pcd *PebbleCollectionDirectory) Open(pebblePath string) error { + db, err := pebble.Open(pebblePath, &pebble.Options{}) + if err != nil { + return fmt.Errorf("%s: could not open db, %w", pebblePath, err) + } + pcd.db = db + pcd.collections = make(map[string]uint32) + pcd.collectionNames = make(map[uint32]string) + if pcd.log == nil { + pcd.log = slog.Default() + } + return pcd.readAllCollectionInterns(context.Background()) +} + +func (pcd *PebbleCollectionDirectory) Close() error { + err := pcd.db.Flush() + if err != nil { + pcd.log.Error("pebble flush", "err", err) + } + err = pcd.db.Close() + if err != nil { + pcd.log.Error("pebble close", "err", err) + } + return err +} + +// readAllCollectionInterns should only be run at setup time inside Open() when locking against threads is not needed +func (pcd *PebbleCollectionDirectory) readAllCollectionInterns(ctx context.Context) error { + lower := []byte{'C'} + upper := []byte{'D'} + iter, err := pcd.db.NewIterWithContext(ctx, &pebble.IterOptions{ + LowerBound: lower, + UpperBound: upper, + }) + if err != nil { + return fmt.Errorf("collection iter start, %w", err) + } + defer iter.Close() + count := 0 + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + value, err := iter.ValueAndErr() + if err != nil { + return fmt.Errorf("collection iter, %w", err) + } + collection := parseCollectionInternKey(key) + collectionId := binary.BigEndian.Uint32(value) + count++ + pcd.collections[collection] = collectionId + pcd.collectionNames[collectionId] = collection + if collectionId > pcd.maxCollectionId { + pcd.maxCollectionId = collectionId + } + pcd.log.Debug("collection", "name", collection, "id", collectionId) + } + pcd.log.Debug("read collections", "count", count, "max", pcd.maxCollectionId) + return nil +} + +type CollectionDidTime struct { + Collection string + Did string + UnixMillis int64 +} + +func (pcd *PebbleCollectionDirectory) ReadAllPrimary(ctx context.Context, out chan<- CollectionDidTime) error { + defer close(out) + lower := []byte{'A'} + upper := []byte{'B'} + iter, err := pcd.db.NewIterWithContext(ctx, &pebble.IterOptions{ + LowerBound: lower, + UpperBound: upper, + }) + if err != nil { + return fmt.Errorf("collection iter start, %w", err) + } + defer iter.Close() + count := 0 + done := ctx.Done() + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + collectionId, did, seenMs := parsePrimaryPebbleRow(key) + count++ + collection := pcd.collectionNames[collectionId] + rec := CollectionDidTime{ + Collection: collection, + Did: did, + UnixMillis: seenMs, + } + select { + case <-done: + return nil + case out <- rec: + } + } + pcd.log.Debug("read primary", "count", count) + return nil +} + +func (pcd *PebbleCollectionDirectory) ReadCollection(ctx context.Context, collection, cursor string, limit int) (result []CollectionDidTime, nextCursor string, err error) { + var lower []byte + collectionId, err := pcd.CollectionToId(collection, false) + if err != nil { + if err == ErrNotFound { + return nil, "", nil + } + return nil, "", fmt.Errorf("collection id err, %w", err) + } + if cursor != "" { + if strings.ContainsAny(cursor, "+/=") { + // NOTE(2025-12-05): this is a temporary flexibility to not break clients using "old" cursor encoding + lower, err = base64.StdEncoding.DecodeString(cursor) + } else { + lower, err = base64.RawURLEncoding.DecodeString(cursor) + } + if err != nil { + return nil, "", fmt.Errorf("could not decode cursor, %w", err) + } + } else { + lower = make([]byte, 1+4) + lower[0] = 'A' + binary.BigEndian.PutUint32(lower[1:], collectionId) + } + var upper [5]byte + upper[0] = 'A' + binary.BigEndian.PutUint32(upper[1:], collectionId+1) + iter, err := pcd.db.NewIterWithContext(ctx, &pebble.IterOptions{ + LowerBound: lower, + UpperBound: upper[:], + }) + if err != nil { + return nil, "", fmt.Errorf("collection iter start, %w", err) + } + defer iter.Close() + count := 0 + done := ctx.Done() + result = make([]CollectionDidTime, 0, limit) + for iter.First(); iter.Valid(); iter.Next() { + key := iter.Key() + collectionId, did, seenMs := parsePrimaryPebbleRow(key) + count++ + collection := pcd.collectionNames[collectionId] + rec := CollectionDidTime{ + Collection: collection, + Did: did, + UnixMillis: seenMs, + } + result = append(result, rec) + breaker := false + if count >= limit { + breaker = true + } else { + select { + case <-done: + breaker = true + default: + } + } + if breaker { + prevKey := make([]byte, len(key), len(key)+1) + copy(prevKey, key) + prevKey = append(prevKey, 0) + nextCursor = base64.RawURLEncoding.EncodeToString(prevKey) + break + } + } + pcd.log.Debug("read primary", "count", count) + return result, nextCursor, nil +} + +var ErrNotFound = errors.New("not found") + +func (pcd *PebbleCollectionDirectory) CollectionToId(collection string, create bool) (uint32, error) { + pcd.collectionsLock.Lock() + defer pcd.collectionsLock.Unlock() + // easy mode: in cache + collectionId, ok := pcd.collections[collection] + if ok { + return collectionId, nil + } + + // read from db + key := makeCollectionInternKey(collection) + value, closer, err := pcd.db.Get(key) + if closer != nil { + defer closer.Close() + } + if err == nil { + collectionId = binary.BigEndian.Uint32(value) + return collectionId, nil + } + + if !create { + return 0, ErrNotFound + } + // make new id, write to db + if errors.Is(err, pebble.ErrNotFound) { + // ok, fall through + } else if err != nil { + return 0, fmt.Errorf("pebble get err, %w", err) + } + collectionId = pcd.maxCollectionId + 1 + pcd.maxCollectionId = collectionId + var cib [4]byte + binary.BigEndian.PutUint32(cib[:], collectionId) + err = pcd.db.Set(key, cib[:], pebble.NoSync) + if err != nil { + return 0, fmt.Errorf("pebble set err, %w", err) + } + pcd.collections[collection] = collectionId + pcd.collectionNames[collectionId] = collection + return collectionId, nil +} + +var trueValue = [1]byte{'t'} + +func (pcd *PebbleCollectionDirectory) CountDidCollections(did string) (int, error) { + lower := make([]byte, 1+len(did)) + lower[0] = 'D' + copy(lower[1:1+len(did)], did) + upper := make([]byte, len(lower)) + copy(upper, lower) + upper[len(upper)-1]++ + ctx := context.Background() + iter, err := pcd.db.NewIterWithContext(ctx, &pebble.IterOptions{ + LowerBound: lower, + UpperBound: upper, + }) + if err != nil { + return 0, fmt.Errorf("did iter start, %w", err) + } + defer iter.Close() + count := 0 + for iter.First(); iter.Valid(); iter.Next() { + //key := iter.Key() + //xdid, xcollectionId := parseByDidKey(key) + count++ + } + return count, nil +} + +func (pcd *PebbleCollectionDirectory) MaybeSetCollection(did, collection string) error { + collectionId, err := pcd.CollectionToId(collection, true) + if err != nil { + return err + } + dkey := makeByDidKey(did, collectionId) + _, closer, err := pcd.db.Get(dkey) + if closer != nil { + defer closer.Close() + } + if err == nil { + // already exists, done + pebbleDup.Inc() + return nil + } + if errors.Is(err, pebble.ErrNotFound) { + // ok, fall through + } else if err != nil { + return fmt.Errorf("pebble get err, %w", err) + } + + now := time.Now() + pkey := makePrimaryPebbleRow(collectionId, did, now.UnixMilli()) + err = pcd.db.Set(pkey, trueValue[:], pebble.NoSync) + if err != nil { + return fmt.Errorf("pebble set err, %w", err) + } + var timebytes [8]byte + binary.BigEndian.PutUint64(timebytes[:], uint64(now.UnixMilli())) + err = pcd.db.Set(dkey, timebytes[:], pebble.NoSync) + if err != nil { + return fmt.Errorf("pebble set err, %w", err) + } + pebbleNew.Inc() + return nil +} + +func (pcd *PebbleCollectionDirectory) SetFromResults(results <-chan DidCollection) { + errcount := 0 + for result := range results { + err := pcd.MaybeSetCollection(result.Did, result.Collection) + if err != nil { + errcount++ + pcd.log.Error("set collection", "err", err) + if errcount > 0 { + // TODO: signal backpressure and shutdown + return + } + } else { + errcount = 0 + } + } +} + +type CollectionStats struct { + CollectionCounts map[string]uint64 `json:"collections"` +} + +func (pcd *PebbleCollectionDirectory) GetCollectionStats() (stats CollectionStats, err error) { + ctx := context.Background() + records := make(chan CollectionDidTime, 1000) + go pcd.ReadAllPrimary(ctx, records) + + stats.CollectionCounts = make(map[string]uint64) + + for rec := range records { + stats.CollectionCounts[rec.Collection]++ + } + + return stats, nil +} + +const seqKey = "Xseq" + +func (pcd *PebbleCollectionDirectory) SetSequence(seq int64) error { + var seqb [8]byte + binary.BigEndian.PutUint64(seqb[:], uint64(seq)) + return pcd.db.Set([]byte(seqKey), seqb[:], pebble.NoSync) +} +func (pcd *PebbleCollectionDirectory) GetSequence() (int64, bool, error) { + vbytes, closer, err := pcd.db.Get([]byte(seqKey)) + if closer != nil { + defer closer.Close() + } + if errors.Is(err, pebble.ErrNotFound) { + return 0, false, nil + } + if err != nil { + return 0, false, fmt.Errorf("pebble seq err, %w", err) + } + seq := int64(binary.BigEndian.Uint64(vbytes)) + return seq, true, nil +} diff --git a/cmd/collectiondir/pebble_test.go b/cmd/collectiondir/pebble_test.go new file mode 100644 index 000000000..608a7a368 --- /dev/null +++ b/cmd/collectiondir/pebble_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "encoding/csv" + "log/slog" + "strings" + "testing" + + "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/vfs" + "github.com/stretchr/testify/assert" +) + +type debugWriter struct { + t *testing.T +} + +func (w *debugWriter) Write(p []byte) (n int, err error) { + w.t.Helper() + w.t.Log(string(p)) + return len(p), nil +} + +// make a new pebble that writes to memory and logs to test.Log +func newMem(t *testing.T) *PebbleCollectionDirectory { + memfs := vfs.NewMem() + db, err := pebble.Open("wat", &pebble.Options{ + FS: memfs, + }) + if err != nil { + panic(err) + } + + log := slog.New(slog.NewTextHandler(&debugWriter{t: t}, &slog.HandlerOptions{Level: slog.LevelDebug})) + pcd := &PebbleCollectionDirectory{ + db: db, + collections: make(map[string]uint32), + collectionNames: make(map[uint32]string), + log: log, + } + if pcd.log == nil { + pcd.log = slog.Default() + } + return pcd +} + +// did, collection +const testDataCsv = `alice,post +alice,like +bob,post +bob,other +carol,post +eve,post +eve,like +eve,other` + +func TestPebbleCollectionDirectory(t *testing.T) { + assert := assert.New(t) + + pcd := newMem(t) + defer func() { + err := pcd.Close() + if err != nil { + t.Error(err) + } + }() + + rows, err := csv.NewReader(strings.NewReader(testDataCsv)).ReadAll() + assert.NoError(err) + for _, row := range rows { + err := pcd.MaybeSetCollection(row[0], row[1]) + assert.NoError(err) + } + stats, err := pcd.GetCollectionStats() + assert.NoError(err) + t.Log(stats) + assert.Equal(uint64(4), stats.CollectionCounts["post"]) + assert.Equal(uint64(2), stats.CollectionCounts["like"]) + assert.Equal(uint64(2), stats.CollectionCounts["other"]) + + t.Log(pcd.collections) + + wat, nextCursor, err := pcd.ReadCollection(context.Background(), "post", "", 1000) + assert.NoError(err) + assert.Equal("", nextCursor) + for _, row := range wat { + assert.Equal("post", row.Collection) + } + assert.Equal(4, len(wat)) + + wat, nextCursor, err = pcd.ReadCollection(context.Background(), "like", "", 1000) + assert.NoError(err) + assert.Equal("", nextCursor) + for _, row := range wat { + assert.Equal("like", row.Collection) + } + assert.Equal(2, len(wat)) + + wat, nextCursor, err = pcd.ReadCollection(context.Background(), "other", "", 1000) + assert.NoError(err) + assert.Equal("", nextCursor) + for _, row := range wat { + assert.Equal("other", row.Collection) + } + assert.Equal(2, len(wat)) +} diff --git a/cmd/collectiondir/serve.go b/cmd/collectiondir/serve.go new file mode 100644 index 000000000..36c6d0ee6 --- /dev/null +++ b/cmd/collectiondir/serve.go @@ -0,0 +1,1035 @@ +package main + +import ( + "compress/gzip" + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + _ "net/http/pprof" + "net/url" + "os" + "os/signal" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/util/svcutil" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/hashicorp/golang-lru/v2" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/urfave/cli/v3" +) + +var serveCmd = &cli.Command{ + Name: "serve", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "api-listen", + Value: ":2510", + Sources: cli.EnvVars("COLLECTIONS_API_LISTEN"), + }, + &cli.StringFlag{ + Name: "metrics-listen", + Value: ":2511", + Sources: cli.EnvVars("COLLECTIONS_METRICS_LISTEN"), + }, + &cli.StringFlag{ + Name: "pebble", + Usage: "path to store pebble db", + Required: true, + }, + &cli.StringFlag{ + Name: "dau-directory", + Usage: "directory to store DAU pebble db", + Required: true, + }, + &cli.StringFlag{ + Name: "upstream", + Usage: "URL, e.g. wss://bsky.network", + Sources: cli.EnvVars("COLLECTIONS_UPSTREAM"), + }, + &cli.StringFlag{ + Name: "admin-token", + Usage: "admin authentication", + Sources: cli.EnvVars("COLLECTIONS_ADMIN_TOKEN"), + }, + &cli.Float64Flag{ + Name: "crawl-qps", + Usage: "per-PDS crawl queries-per-second limit", + Value: 100, + }, + &cli.StringFlag{ + Name: "ratelimit-header", + Usage: "secret for friend PDSes", + Sources: cli.EnvVars("BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"), + }, + &cli.Uint64Flag{ + Name: "clist-min-dids", + Usage: "filter collection list to >= N dids", + Value: 5, + Sources: cli.EnvVars("COLLECTIONS_CLIST_MIN_DIDS"), + }, + &cli.IntFlag{ + Name: "max-did-collections", + Usage: "stop recording new collections per did after it has >= this many collections", + Value: 1000, + Sources: cli.EnvVars("COLLECTIONS_MAX_DID_COLLECTIONS"), + }, + &cli.StringFlag{ + Name: "sets-json-path", + Usage: "file path of JSON file containing static word sets", + Sources: cli.EnvVars("HEPA_SETS_JSON_PATH", "COLLECTIONS_SETS_JSON_PATH"), + }, + &cli.BoolFlag{ + Name: "verbose", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + var server collectionServer + return server.run(ctx, cmd) + }, +} + +type BadwordChecker interface { + HasBadword(string) bool +} + +type collectionServer struct { + ctx context.Context + + // the primary directory, all repos ever and their collections + pcd *PebbleCollectionDirectory + + // daily-active-user directory, new directory every 00:00:00 UTC + dauDirectory *PebbleCollectionDirectory + dauDirectoryPath string // currently open dauDirectory, {dauDirectoryDir}/{YYYY}{mm}{dd}.pebble + dauDay time.Time // YYYY-MM-DD 00:00:00 UTC + dauTomorrow time.Time + dauDirectoryDir string + + statsCache *CollectionStats + statsCacheWhen time.Time + statsCacheLock sync.Mutex + statsCacheFresh sync.Cond + statsCachePending bool + + // (did,collection) pairs from firehose + ingestFirehose chan DidCollection + // (did,collection) pairs from PDS crawl (don't apply to dauDirectory) + ingestCrawl chan DidCollection + + log *slog.Logger + + AdminToken string + ExepctedAuthHeader string + PerPDSCrawlQPS float64 + + activeCrawls map[string]activeCrawl + activeCrawlsLock sync.Mutex + + shutdown chan struct{} + + wg sync.WaitGroup + + ratelimitHeader string + + apiServer *http.Server + metricsServer *http.Server + + MinDidsForCollectionList uint64 + MaxDidCollections int + + didCollectionCounts *lru.Cache[string, int] + + badwords BadwordChecker +} + +type activeCrawl struct { + start time.Time + stats *CrawlStats +} + +func (cs *collectionServer) run(ctx context.Context, cmd *cli.Command) error { + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + cs.shutdown = make(chan struct{}) + level := slog.LevelInfo + if cmd.Bool("verbose") { + level = slog.LevelDebug + } + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})) + slog.SetDefault(log) + + if cmd.IsSet("ratelimit-header") { + cs.ratelimitHeader = cmd.String("ratelimit-header") + } + if cmd.IsSet("sets-json-path") { + badwords, err := loadBadwords(cmd.String("sets-json-path")) + if err != nil { + return err + } + cs.badwords = badwords + } + cs.MinDidsForCollectionList = cmd.Uint64("clist-min-dids") + cs.MaxDidCollections = cmd.Int("max-did-collections") + cs.ingestFirehose = make(chan DidCollection, 1000) + cs.ingestCrawl = make(chan DidCollection, 1000) + var err error + cs.didCollectionCounts, err = lru.New[string, int](1_000_000) // TODO: configurable LRU size + if err != nil { + return fmt.Errorf("lru init, %w", err) + } + cs.log = log + cs.ctx = ctx + cs.AdminToken = cmd.String("admin-token") + cs.ExepctedAuthHeader = "Bearer " + cs.AdminToken + cs.wg.Add(1) + go cs.ingestReceiver() + pebblePath := cmd.String("pebble") + cs.pcd = &PebbleCollectionDirectory{ + log: cs.log, + } + err = cs.pcd.Open(pebblePath) + if err != nil { + return fmt.Errorf("%s: failed to open pebble db: %w", pebblePath, err) + } + cs.dauDirectoryDir = cmd.String("dau-directory") + if cs.dauDirectoryDir != "" { + err := cs.openDau() + if err != nil { + return err + } + } + cs.statsCacheFresh.L = &cs.statsCacheLock + + apiServerEcho, err := cs.createApiServer(ctx, cmd.String("api-listen")) + if err != nil { + return err + } + cs.wg.Add(1) + go func() { cs.StartApiServer(ctx, apiServerEcho) }() + + cs.createMetricsServer(cmd.String("metrics-listen")) + cs.wg.Add(1) + go func() { cs.StartMetricsServer(ctx) }() + + upstream := cmd.String("upstream") + if upstream != "" { + fh := Firehose{ + Log: log, + Host: upstream, + Seq: -1, + } + seq, seqok, err := cs.pcd.GetSequence() + if err != nil { + cs.log.Warn("db get seq", "err", err) + } else if seqok { + fh.Seq = seq + } + fhevents := make(chan *events.XRPCStreamEvent, 1000) + cs.wg.Add(1) + go cs.firehoseThread(&fh, fhevents) + cs.wg.Add(1) + go cs.handleFirehose(fhevents) + } + + <-signals + log.Info("received shutdown signal") + return cs.Shutdown() +} + +func (cs *collectionServer) openDau() error { + now := time.Now().UTC() + ymd := now.Format("2006-01-02") + fname := fmt.Sprintf("d%s.pebble", ymd) + fpath := filepath.Join(cs.dauDirectoryDir, fname) + daud := &PebbleCollectionDirectory{ + log: cs.log, + } + err := daud.Open(fpath) + if err != nil { + return fmt.Errorf("%s: failed to open dau pebble db: %w", fpath, err) + } + cs.dauDirectory = daud + cs.dauDirectoryPath = fpath + cs.dauDay = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + cs.dauTomorrow = cs.dauDay.AddDate(0, 0, 1) + cs.log.Info("DAU db opened", "path", fpath) + return nil +} + +func (cs *collectionServer) Shutdown() error { + close(cs.shutdown) + + func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + cs.log.Info("metrics shutdown start") + sherr := cs.metricsServer.Shutdown(ctx) + cs.log.Info("metrics shutdown", "err", sherr) + }() + + func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cs.log.Info("api shutdown start...") + err := cs.apiServer.Shutdown(ctx) + cs.log.Info("api shutdown, thread wait...", "err", err) + }() + + cs.log.Info("threads done, db close...") + err := cs.pcd.Close() + if err != nil { + cs.log.Error("failed to shutdown pebble", "err", err) + } + cs.log.Info("db done. done.") + cs.wg.Wait() + return err +} + +// firehoseThreads is responsible for connecting to upstream firehose source +func (cs *collectionServer) firehoseThread(fh *Firehose, fhevents chan<- *events.XRPCStreamEvent) { + defer cs.wg.Done() + defer cs.log.Info("firehoseThread exit") + ctx, cancel := context.WithCancel(cs.ctx) + go func() { + <-cs.shutdown + cancel() + }() + err := fh.subscribeWithRedialer(ctx, fhevents) + if err != nil { + cs.log.Error("failed to subscribe to redialer", "err", err) + } + if fh.Seq >= 0 { + err := cs.pcd.SetSequence(fh.Seq) + if err != nil { + cs.log.Warn("db set seq", "err", err) + } + } +} + +// handleFirehose consumes XRPCStreamEvent from firehoseThread(), further parses data and applies +func (cs *collectionServer) handleFirehose(fhevents <-chan *events.XRPCStreamEvent) { + defer cs.wg.Done() + defer cs.log.Info("handleFirehose exit") + defer close(cs.ingestFirehose) + var lastSeq int64 + lastSeqSet := false + notDone := true + for notDone { + select { + case <-cs.shutdown: + cs.log.Info("firehose handler shutdown") + notDone = false + case evt, ok := <-fhevents: + if !ok { + notDone = false + cs.log.Info("firehose handler closed") + break + } + firehoseReceivedCounter.Inc() + seq, ok := evt.GetSequence() + if ok { + lastSeq = seq + lastSeqSet = true + } + if evt.RepoCommit != nil { + firehoseCommits.Inc() + cs.handleCommit(evt.RepoCommit) + } + } + } + if lastSeqSet { + cs.pcd.SetSequence(lastSeq) + } +} + +func (cs *collectionServer) handleCommit(commit *comatproto.SyncSubscribeRepos_Commit) { + for _, op := range commit.Ops { + // op.Path is collection/rkey + nsid, _, err := syntax.ParseRepoPath(op.Path) + if err != nil { + cs.log.Warn("bad op path", "repo", commit.Repo, "err", err) + return + } + firehoseCommitOps.WithLabelValues(op.Action).Inc() + if op.Action == "create" || op.Action == "update" { + firehoseDidcSet.Inc() + cs.ingestFirehose <- DidCollection{ + Did: commit.Repo, + Collection: nsid.String(), + } + } + } +} + +func (cs *collectionServer) createMetricsServer(addr string) { + e := echo.New() + e.GET("/metrics", echo.WrapHandler(promhttp.Handler())) + e.Any("/debug/pprof/*", echo.WrapHandler(http.DefaultServeMux)) + + cs.metricsServer = &http.Server{ + Addr: addr, + Handler: e, + } +} + +func (cs *collectionServer) StartMetricsServer(ctx context.Context) { + defer cs.wg.Done() + defer cs.log.Info("metrics server exit") + + err := cs.metricsServer.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + slog.Error("error in metrics server", "err", err) + os.Exit(1) + } +} + +func (cs *collectionServer) createApiServer(ctx context.Context, addr string) (*echo.Echo, error) { + var lc net.ListenConfig + li, err := lc.Listen(ctx, "tcp", addr) + if err != nil { + return nil, err + } + e := echo.New() + e.HideBanner = true + + e.Use(svcutil.MetricsMiddleware) + e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ + AllowOrigins: []string{"*"}, + AllowHeaders: []string{echo.HeaderOrigin, echo.HeaderContentType, echo.HeaderAccept, echo.HeaderAuthorization}, + })) + + e.GET("/_health", cs.healthz) + + e.GET("/xrpc/com.atproto.sync.listReposByCollection", cs.getDidsForCollection) + e.GET("/v1/getDidsForCollection", cs.getDidsForCollection) + e.GET("/v1/listCollections", cs.listCollections) + + // TODO: allow public 'requestCrawl' API? + //e.GET("/xrpc/com.atproto.sync.requestCrawl", cs.crawlPds) + //e.POST("/xrpc/com.atproto.sync.requestCrawl", cs.crawlPds) + + // admin auth heador required + e.POST("/admin/pds/requestCrawl", cs.crawlPds) // same as relay + e.GET("/admin/crawlStatus", cs.crawlStatus) + + e.Listener = li + srv := &http.Server{ + Handler: e, + } + cs.apiServer = srv + return e, nil +} + +func (cs *collectionServer) StartApiServer(ctx context.Context, e *echo.Echo) { + defer cs.wg.Done() + defer cs.log.Info("api server exit") + err := cs.apiServer.Serve(e.Listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + slog.Error("error in api server", "err", err) + os.Exit(1) + } +} + +const statsCacheDuration = time.Second * 300 + +func getLimit(c echo.Context, min, defaultLim, max int) int { + limstr := c.QueryParam("limit") + if limstr == "" { + return defaultLim + } + lvx, err := strconv.ParseInt(limstr, 10, 64) + if err != nil { + return defaultLim + } + lv := int(lvx) + if lv < min { + return min + } + if lv > max { + return max + } + return lv +} + +// /xrpc/com.atproto.sync.listReposByCollection?collection={}&cursor={}&limit={50<=N<=1000} +// /v1/getDidsForCollection?collection={}&cursor={}&limit={50<=N<=1000} +// +// returns +// {"dids":["did:A", "..."], "cursor":"opaque text"} +func (cs *collectionServer) getDidsForCollection(c echo.Context) error { + ctx := c.Request().Context() + collection := c.QueryParam("collection") + _, err := syntax.ParseNSID(collection) + if err != nil { + return c.JSON(http.StatusBadRequest, xrpc.XRPCError{ErrStr: "BadRequest", Message: fmt.Sprintf("bad collection nsid, %s", err.Error())}) + } + cursor := c.QueryParam("cursor") + limit := getLimit(c, 1, 500, 10_000) + they, nextCursor, err := cs.pcd.ReadCollection(ctx, collection, cursor, limit) + if err != nil { + slog.Error("ReadCollection", "collection", collection, "cursor", cursor, "limit", limit, "err", err) + return c.JSON(http.StatusInternalServerError, xrpc.XRPCError{ErrStr: "DatabaseError", Message: "failed to read DIDs for collection"}) + } + cs.log.Info("getDidsForCollection", "collection", collection, "cursor", cursor, "limit", limit, "count", len(they), "nextCursor", nextCursor) + var out comatproto.SyncListReposByCollection_Output + out.Repos = make([]*comatproto.SyncListReposByCollection_Repo, len(they)) + for i, rec := range they { + out.Repos[i] = &comatproto.SyncListReposByCollection_Repo{Did: rec.Did} + } + if nextCursor != "" { + out.Cursor = &nextCursor + } + return c.JSON(http.StatusOK, out) +} + +// return cached collection stats if they're fresh +// return new collection stats if they can be calculated quickly +// return stale cached collection stats if new stats take too long +// just wait for fresh stats if there are no cached stats +// stalenessAllowed is how old stats can be before we try to recalculate them, 0=default of 5 minutes +func (cs *collectionServer) getStatsCache(stalenessAllowed time.Duration) (*CollectionStats, error) { + if stalenessAllowed <= 0 { + stalenessAllowed = statsCacheDuration + } + var statsCache *CollectionStats + var staleCache *CollectionStats + var waiter *freshStatsWaiter + cs.statsCacheLock.Lock() + if cs.statsCache != nil { + if time.Since(cs.statsCacheWhen) < stalenessAllowed { + // has fresh! + statsCache = cs.statsCache + } else if !cs.statsCachePending { + cs.statsCachePending = true + go cs.statsBuilder() + staleCache = cs.statsCache + } else { + staleCache = cs.statsCache + } + if staleCache != nil { + waiter = &freshStatsWaiter{ + cs: cs, + freshCache: make(chan *CollectionStats), + } + go waiter.waiter() + } + } else if !cs.statsCachePending { + cs.statsCachePending = true + go cs.statsBuilder() + } + cs.statsCacheLock.Unlock() + + if statsCache != nil { + // return fresh-enough data + return statsCache, nil + } + + if staleCache == nil { + // block forever waiting for fresh data + cs.statsCacheLock.Lock() + for cs.statsCache == nil { + cs.statsCacheFresh.Wait() + } + statsCache = cs.statsCache + cs.statsCacheLock.Unlock() + return statsCache, nil + } + + // wait for up to a second for fresh data, on timeout return stale data + timeout := time.NewTimer(time.Second) + defer timeout.Stop() + select { + case <-timeout.C: + cs.statsCacheLock.Lock() + waiter.l.Lock() + waiter.obsolete = true + waiter.l.Unlock() + cs.statsCacheLock.Unlock() + return staleCache, nil + case statsCache = <-waiter.freshCache: + return statsCache, nil + } +} + +type freshStatsWaiter struct { + cs *collectionServer + l sync.Mutex + obsolete bool + freshCache chan *CollectionStats +} + +func (fsw *freshStatsWaiter) waiter() { + fsw.cs.statsCacheLock.Lock() + defer fsw.cs.statsCacheLock.Unlock() + fsw.cs.statsCacheFresh.Wait() + fsw.l.Lock() + defer fsw.l.Unlock() + if fsw.obsolete { + close(fsw.freshCache) + } else { + fsw.freshCache <- fsw.cs.statsCache + } +} + +func (cs *collectionServer) statsBuilder() { + for { + start := time.Now() + stats, err := cs.pcd.GetCollectionStats() + dt := time.Since(start) + if err == nil { + statsCalculations.Observe(dt.Seconds()) + countsum := uint64(0) + for _, v := range stats.CollectionCounts { + countsum += v + } + cs.log.Info("stats built", "dt", dt, "total", countsum) + cs.statsCacheLock.Lock() + cs.statsCache = &stats + cs.statsCacheWhen = time.Now() + cs.statsCacheFresh.Broadcast() + cs.statsCachePending = false + cs.statsCacheLock.Unlock() + return + } else { + cs.log.Error("GetCollectionStats", "dt", dt, "err", err) + time.Sleep(2 * time.Second) + } + } +} + +func (cs *collectionServer) hasBadword(collection string) bool { + if cs.badwords != nil { + return cs.badwords.HasBadword(collection) + } + return false +} + +// /v1/listCollections?c={}&cursor={}&limit={50<=limit<=1000} +// +// admin may set ?stalesec={} for a maximum number of seconds stale data is accepted +// +// returns +// {"collections":{"app.bsky.feed.post": 123456789, "some collection": 42}, "cursor":"opaque text"} +func (cs *collectionServer) listCollections(c echo.Context) error { + stalenessAllowed := statsCacheDuration + stalesecStr := c.QueryParam("stalesec") + if stalesecStr != "" && cs.isAdmin(c) { + stalesec, err := strconv.ParseInt(stalesecStr, 10, 64) + if err != nil { + return c.JSON(http.StatusBadRequest, xrpc.XRPCError{ErrStr: "BadRequest", Message: "invalid 'stalesec' query parameter"}) + } + if stalesec == 0 { + stalenessAllowed = 1 + } else { + stalenessAllowed = time.Duration(stalesec) * time.Second + } + cs.log.Info("stalesec", "q", stalesecStr, "d", stalenessAllowed) + } + stats, err := cs.getStatsCache(stalenessAllowed) + if err != nil { + slog.Error("getStatsCache", "err", err) + return c.JSON(http.StatusInternalServerError, xrpc.XRPCError{ErrStr: "DatabaseError", Message: "failed to read stats"}) + } + cursor := c.QueryParam("cursor") + collections, hasQueryCollections := c.QueryParams()["c"] + limit := getLimit(c, 50, 500, 1000) + var out ListCollectionsResponse + if hasQueryCollections { + out.Collections = make(map[string]uint64, len(collections)) + for _, collection := range collections { + count, ok := stats.CollectionCounts[collection] + if ok { + out.Collections[collection] = count + } + } + } else { + allCollections := make([]string, 0, len(stats.CollectionCounts)) + for collection := range stats.CollectionCounts { + allCollections = append(allCollections, collection) + } + sort.Strings(allCollections) + out.Collections = make(map[string]uint64, limit) + count := 0 + for _, collection := range allCollections { + if (cursor == "") || (collection > cursor) { + if cs.hasBadword(collection) { + // don't show badwords in public list of collections + continue + } + if stats.CollectionCounts[collection] < cs.MinDidsForCollectionList { + // don't show experimental/spam collections only implemented by a few DIDs + continue + } + // TODO: probably regex based filter for collection-spam + out.Collections[collection] = stats.CollectionCounts[collection] + count++ + if count >= limit { + out.Cursor = collection + } + } + } + } + return c.JSON(http.StatusOK, out) +} + +type ListCollectionsResponse struct { + Collections map[string]uint64 `json:"collections"` + Cursor string `json:"cursor"` +} + +func (cs *collectionServer) ingestReceiver() { + defer cs.wg.Done() + defer cs.log.Info("ingestReceiver exit") + errcount := 0 + for { + select { + case didc, ok := <-cs.ingestFirehose: + if !ok { + cs.log.Info("ingestFirehose closed") + return + } + err := cs.ingestDidc(didc, true) + if err != nil { + errcount++ + } else { + errcount = 0 + } + case didc := <-cs.ingestCrawl: + err := cs.ingestDidc(didc, false) + if err != nil { + errcount++ + } else { + errcount = 0 + } + case <-cs.shutdown: + cs.log.Info("shutting down ingestReceiver") + return + } + if errcount > 10 { + cs.log.Error("ingestReceiver too many errors") + return // TODO: cancel parent somehow + } + } +} + +func (cs *collectionServer) ingestDidc(didc DidCollection, dau bool) error { + count, ok := cs.didCollectionCounts.Get(didc.Did) + var err error + if !ok { + count, err = cs.pcd.CountDidCollections(didc.Did) + if err != nil { + return fmt.Errorf("count did collections, %s %w", didc.Did, err) + } + cs.didCollectionCounts.Add(didc.Did, count) + } + if count >= cs.MaxDidCollections { + cs.log.Warn("did too many collections", "did", didc.Did) + return nil + } + err = cs.pcd.MaybeSetCollection(didc.Did, didc.Collection) + if err != nil { + cs.log.Warn("pcd write", "err", err) + return err + } + if dau && cs.dauDirectory != nil { + err = cs.maybeDauWrite(didc) + if err != nil { + cs.log.Warn("dau write", "err", err) + return err + } + } + return nil +} + +func (cs *collectionServer) maybeDauWrite(didc DidCollection) error { + now := time.Now() + if now.After(cs.dauTomorrow) { + go dauStats(cs.dauDirectory, cs.dauDay, cs.dauDirectoryDir, cs.log) + cs.dauDirectory = nil + err := cs.openDau() + if err != nil { + return fmt.Errorf("dau reopen, %w", err) + } + } + return cs.dauDirectory.MaybeSetCollection(didc.Did, didc.Collection) +} + +// write {dauDirectoryDir}/d{YYYY-MM-DD}.pebble stats summary to {dauDirectoryDir}/d{YYYY-MM-DD}.csv.gz +func dauStats(oldDau *PebbleCollectionDirectory, dauDay time.Time, dauDir string, log *slog.Logger) { + fname := fmt.Sprintf("d%s.csv.gz", dauDay.Format("2006-01-02")) + outstatsPath := filepath.Join(dauDir, fname) + log = log.With("path", outstatsPath) + log.Info("DAU stats summarize") + stats, err := oldDau.GetCollectionStats() + e2 := oldDau.Close() + if e2 != nil { + log.Error("old DAU close", "err", e2) + } + if err != nil { + log.Error("old DAU stats", "err", err) + } else { + log.Info("DAU stats summarized", "rows", len(stats.CollectionCounts)) + pcdStatsToCsvGz(stats, outstatsPath, log) + } +} + +func pcdStatsToCsvGz(stats CollectionStats, outpath string, log *slog.Logger) { + fout, err := os.Create(outpath) + if err != nil { + log.Error("DAU stats open", "err", err) + return + } + defer fout.Close() + gzout := gzip.NewWriter(fout) + defer gzout.Close() + csvout := csv.NewWriter(gzout) + defer csvout.Flush() + err = csvout.Write([]string{"collection", "count"}) + if err != nil { + log.Error("DAU stats header", "err", err) + return + } + var row [2]string + rowcount := 0 + for collection, count := range stats.CollectionCounts { + row[0] = collection + row[1] = strconv.FormatUint(count, 10) + err = csvout.Write(row[:]) + if err != nil { + log.Error("DAU stats row", "err", err) + return + } + rowcount++ + } + log.Info("DAU stats ok", "rows", rowcount) +} + +type CrawlRequest struct { + Host string `json:"hostname,omitempty"` + Hosts []string `json:"hosts,omitempty"` +} + +type CrawlRequestResponse struct { + Message string `json:"message,omitempty"` + Error string `json:"error,omitempty"` +} + +func hostOrUrlToUrl(host string) string { + xu, err := url.Parse(host) + if err != nil { + xu = new(url.URL) + xu.Host = host + xu.Scheme = "https" + return xu.String() + } else if xu.Scheme == "" { + xu.Scheme = "https" + return xu.String() + } + return host +} + +func (cs *collectionServer) isAdmin(c echo.Context) bool { + authHeader := c.Request().Header.Get("Authorization") + if authHeader == "" { + return false + } + if authHeader == cs.ExepctedAuthHeader { + return true + } + cs.log.Info("wrong auth header", "header", authHeader, "expected", cs.ExepctedAuthHeader) + return false +} + +// /admin/pds/requestCrawl +// same API signature as relay admin requestCrawl +// starts a crawl and returns. See /v1/crawlStatus +// requires header `Authorization: Bearer {admin token}` +// +// POST {"hostname":"one hostname or URL", "hosts":["up to 1000 hosts", "..."]} +// OR +// POST /admin/pds/requestCrawl?hostname={one host} +func (cs *collectionServer) crawlPds(c echo.Context) error { + isAdmin := cs.isAdmin(c) + if !isAdmin { + return c.JSON(http.StatusForbidden, xrpc.XRPCError{ErrStr: "AdminRequired", Message: "this endpoint requires admin auth"}) + } + hostQ := c.QueryParam("host") + if hostQ != "" { + go cs.crawlThread(hostQ) + return c.JSON(http.StatusOK, CrawlRequestResponse{Message: "ok"}) + } + + var req CrawlRequest + err := c.Bind(&req) + if err != nil { + cs.log.Info("bad crawl bind", "err", err) + return c.JSON(http.StatusBadRequest, xrpc.XRPCError{ErrStr: "BadRequest", Message: fmt.Sprintf("failed to parse body: %s", err)}) + } + if req.Host != "" { + go cs.crawlThread(req.Host) + } + for _, host := range req.Hosts { + go cs.crawlThread(host) + } + return c.JSON(http.StatusOK, CrawlRequestResponse{Message: "ok"}) +} + +func (cs *collectionServer) crawlThread(hostIn string) { + host := hostOrUrlToUrl(hostIn) + if host != hostIn { + cs.log.Info("going to crawl", "in", hostIn, "as", host) + } + httpClient := http.Client{} + rpcClient := xrpc.Client{ + Host: host, + Client: &httpClient, + } + if cs.ratelimitHeader != "" { + rpcClient.Headers = map[string]string{ + "x-ratelimit-bypass": cs.ratelimitHeader, + } + } + crawler := Crawler{ + Ctx: cs.ctx, + RpcClient: &rpcClient, + QPS: cs.PerPDSCrawlQPS, + Results: cs.ingestCrawl, + Log: cs.log, + } + start := time.Now() + ok, crawlStats := cs.recordCrawlStart(host, start) + if !ok { + cs.log.Info("not crawling dup", "host", host) + return + } + crawler.Stats = crawlStats + cs.log.Info("crawling", "host", host) + err := crawler.CrawlPDSRepoCollections() + cs.clearActiveCrawl(host) + pdsCrawledCounter.Inc() + if err != nil { + cs.log.Warn("crawl err", "host", host, "err", err) + } else { + dt := time.Since(start) + cs.log.Info("crawl done", "host", host, "dt", dt) + } +} + +// recordCrawlStart returns true if ok, false if duplicate +func (cs *collectionServer) recordCrawlStart(host string, start time.Time) (ok bool, stats *CrawlStats) { + cs.activeCrawlsLock.Lock() + defer cs.activeCrawlsLock.Unlock() + if cs.activeCrawls == nil { + cs.activeCrawls = make(map[string]activeCrawl) + } else { + _, dup := cs.activeCrawls[host] + if dup { + return false, nil + } + } + stats = new(CrawlStats) + cs.activeCrawls[host] = activeCrawl{ + start: start, + stats: stats, + } + return true, stats +} + +func (cs *collectionServer) clearActiveCrawl(host string) { + cs.activeCrawlsLock.Lock() + defer cs.activeCrawlsLock.Unlock() + if cs.activeCrawls == nil { + return + } + delete(cs.activeCrawls, host) +} + +type CrawlStatusResponse struct { + HostCrawls map[string]HostCrawl `json:"host_starts"` + ServerTime string `json:"server_time"` +} +type HostCrawl struct { + Start string `json:"start"` + ReposDescribed uint32 `json:"seen"` +} + +// GET /v1/crawlStatus +func (cs *collectionServer) crawlStatus(c echo.Context) error { + authHeader := c.Request().Header.Get("Authorization") + if authHeader != cs.ExepctedAuthHeader { + return c.JSON(http.StatusBadRequest, xrpc.XRPCError{ErrStr: "AdminAuthRequired", Message: "this endpoint requires admin-level auth"}) + } + var out CrawlStatusResponse + out.HostCrawls = make(map[string]HostCrawl) + cs.activeCrawlsLock.Lock() + defer cs.activeCrawlsLock.Unlock() + for host, rec := range cs.activeCrawls { + start := rec.start + out.HostCrawls[host] = HostCrawl{ + Start: start.UTC().Format(time.RFC3339Nano), + ReposDescribed: rec.stats.ReposDescribed.Load(), + } + } + out.ServerTime = time.Now().UTC().Format(time.RFC3339Nano) + return c.JSON(http.StatusOK, out) +} + +func (cs *collectionServer) healthz(c echo.Context) error { + // TODO: check database or upstream health? + return c.JSON(http.StatusOK, map[string]any{"status": "ok"}) +} + +func loadBadwords(path string) (*BadwordsRE, error) { + fin, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("%s: could not open badwords, %w", path, err) + } + dec := json.NewDecoder(fin) + var rules map[string][]string + err = dec.Decode(&rules) + if err != nil { + return nil, fmt.Errorf("%s: badwords json, %w", path, err) + } + + // compile a regex to search a string for any instance of a bad word, because we're expecting things runpooptogether + badwords := rules["worst-words"] + rwords := make([]string, len(badwords)) + for i, word := range badwords { + rwords[i] = regexp.QuoteMeta(word) + } + reStr := strings.Join(rwords, "|") + re, err := regexp.Compile(reStr) + if err != nil { + return nil, fmt.Errorf("%s: badwords regex, %w", path, err) + } + return &BadwordsRE{re: re}, nil +} + +type BadwordsRE struct { + re *regexp.Regexp +} + +func (bw *BadwordsRE) HasBadword(s string) bool { + // TODO: if this is too slow, try more specialized algorithm e.g. https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm + return bw.re.FindString(s) != "" +} diff --git a/cmd/fakermaker/README.md b/cmd/fakermaker/README.md new file mode 100644 index 000000000..a2cb6073a --- /dev/null +++ b/cmd/fakermaker/README.md @@ -0,0 +1,54 @@ + +## Running `fakermaker` + +Configure a `.env` file for use against `atproto` Typescript PDS implementation +(in development mode, already running locally): + + ATP_PDS_HOST=http://localhost:2583 + ATP_AUTH_HANDLE="admin.test" + ATP_AUTH_PASSWORD="admin" + ATP_AUTH_ADMIN_PASSWORD="admin" + + +Then, from the top-level directory, run test commands: + + mkdir -p data/fakermaker + export GOLOG_LOG_LEVEL=info + + # setup and create initial accounts; 100 by default + # supply --use-invite-code and/or --domain-suffix SUFFIX as needed + go run ./cmd/fakermaker/ gen-accounts > data/fakermaker/accounts.json + + # create or update profiles for all the accounts + go run ./cmd/fakermaker/ gen-profiles + + # create follow graph between accounts + go run ./cmd/fakermaker/ gen-graph + + # create posts, including mentions and image uploads + go run ./cmd/fakermaker/ gen-posts + + # create more interactions, such as likes, between accounts + go run ./cmd/fakermaker/ gen-interactions + + # lastly, read-only queries, including timelines, notifications, and post threads + go run ./cmd/fakermaker/ run-browsing + + +## Docker Compose Integration Tests + +To run against Typescript services running in Docker, use the docker compose +file in this directory. + +Run all the servics: + + docker-compose up + +Then configure and run `fakermaker` using the commands above. To run automated integration tests: + + # from top-level directory of this repo + make test-interop + +If you need to wipe volumes (all databases): + + docker-compose down -v diff --git a/cmd/fakermaker/docker-compose.yaml b/cmd/fakermaker/docker-compose.yaml new file mode 100644 index 000000000..699a62fa9 --- /dev/null +++ b/cmd/fakermaker/docker-compose.yaml @@ -0,0 +1,125 @@ +version: '3' +volumes: + dbvol: + driver: local +services: + db: + image: postgres:14.3-alpine + restart: always + environment: + - POSTGRES_USER=bsky + - POSTGRES_PASSWORD=yksb + # intentionally *not* exposing default postgresql port to host (conflicts with other local dev DBs) + command: -p 6543 + ports: + - '6543:6543' + expose: + - '6543' + volumes: + - dbvol:/var/lib/postgresql/docker + - ./docker-create-dev-dbs.sql:/docker-entrypoint-initdb.d/create-dev-dbs.sql + healthcheck: + test: ["CMD-SHELL", "pg_isready -p 6543 -U bsky"] + interval: 3s + timeout: 2s + retries: 3 + + plc: + image: ghcr.io/bluesky-social/did-method-plc:plc-c54aea0373e65df0b87f5bc81710007092f539b1 # DID doc iteration branch + ports: + - "2582:2582" + expose: + - "2582" + environment: + - ENV=dev + - DATABASE_URL=postgres://bsky:yksb@db:6543/plc_dev + - PORT=2582 + - DEBUG_MODE=1 + - LOG_ENABLED=true + - LOG_LEVEL=info + - LOG_DESTINATION=1 + working_dir: /app/packages/server + command: yarn run start + links: + - db + depends_on: + db: + condition: service_healthy + + pds-one: + image: ghcr.io/bluesky-social/atproto:pds-19d2bdc4576bfabe6609afe160cc2c220c351579 + ports: + - "2583:2583" + expose: + - "2583" + environment: + - ENV=dev + - DB_POSTGRES_URL=postgres://bsky:yksb@db:6543/pds_dev + - AVAILABLE_USER_DOMAINS=.test,.dev.bsky.dev + - DID_PLC_URL=http://plc:2582 + - PORT=2583 + - DEBUG_MODE=1 + - LOG_LEVEL=info + - LOG_ENABLED=true + - LOG_DESTINATION=1 + working_dir: /app/packages/pds + command: yarn run start + links: + - db + - plc + depends_on: + db: + condition: service_healthy + plc: + condition: service_started + + bgs: + # to build/run locally, uncomment this instead + #build: ../bigsky/ + image: ghcr.io/bluesky-social/indigo:bigsky-ed809269938c453ebed3a05487acc6417d77bb4e + ports: + - "2470:2470" + environment: + - ATP_PLC_HOST=http://plc:2582 + - ATP_PDS_HOST=http://pds-one:2583 + - GOLOG_LOG_LEVEL=info + links: + - db + - plc + - pds-one + depends_on: + db: + condition: service_healthy + plc: + condition: service_started + pds-one: + condition: service_started + + appview: + image: ghcr.io/bluesky-social/atproto:bsky-19d2bdc4576bfabe6609afe160cc2c220c351579 + ports: + - "2584:2584" + expose: + - "2584" + environment: + - ENV=dev + - DB_POSTGRES_URL=postgres://bsky:yksb@db:6543/bsky_dev + - DID_PLC_URL=http://plc:2582 + - PORT=2584 + - DEBUG_MODE=1 + - LOG_LEVEL=info + - LOG_ENABLED=true + - LOG_DESTINATION=1 + working_dir: /app/packages/bsky + command: yarn run start + links: + - db + - plc + - bgs + depends_on: + db: + condition: service_healthy + plc: + condition: service_started + bgs: + condition: service_started diff --git a/cmd/fakermaker/docker-create-dev-dbs.sql b/cmd/fakermaker/docker-create-dev-dbs.sql new file mode 100644 index 000000000..148cc3e96 --- /dev/null +++ b/cmd/fakermaker/docker-create-dev-dbs.sql @@ -0,0 +1,5 @@ + +CREATE DATABASE plc_dev; +CREATE DATABASE pds_dev; +CREATE DATABASE bgs_dev; +CREATE DATABASE bsky_dev; diff --git a/cmd/fakermaker/main.go b/cmd/fakermaker/main.go new file mode 100644 index 000000000..5dca3c960 --- /dev/null +++ b/cmd/fakermaker/main.go @@ -0,0 +1,494 @@ +// Tool to generate fake accounts, content, and interactions. +// Intended for development and benchmarking. Similar to 'stress' and could +// merge at some point. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "runtime" + + _ "github.com/joho/godotenv/autoload" + _ "go.uber.org/automaxprocs" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/fakedata" + "github.com/bluesky-social/indigo/util/cliutil" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/urfave/cli/v3" + "golang.org/x/sync/errgroup" +) + +func main() { + run(os.Args) +} + +func run(args []string) { + + app := cli.Command{ + Name: "fakermaker", + Usage: "bluesky fake account/content generator", + Version: versioninfo.Short(), + } + + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "pds-host", + Usage: "method, hostname, and port of PDS instance", + Value: "http://localhost:4849", + Sources: cli.EnvVars("ATP_PDS_HOST"), + }, + &cli.StringFlag{ + Name: "admin-password", + Usage: "admin authentication password for PDS", + Required: true, + Sources: cli.EnvVars("ATP_AUTH_ADMIN_PASSWORD"), + }, + &cli.IntFlag{ + Name: "jobs", + Aliases: []string{"j"}, + Usage: "number of parallel threads to use", + Value: runtime.NumCPU(), + }, + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "gen-accounts", + Usage: "create accounts (DID, handle, profile)", + Action: genAccounts, + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "count", + Aliases: []string{"n"}, + Usage: "total number of accounts to create", + Value: 100, + }, + &cli.IntFlag{ + Name: "count-celebrities", + Usage: "number of accounts as 'celebrities' (many followers)", + Value: 10, + }, + &cli.StringFlag{ + Name: "domain-suffix", + Usage: "domain to register handle under", + Value: "test", + }, + &cli.BoolFlag{ + Name: "use-invite-code", + Usage: "create and use an invite code", + Value: false, + }, + }, + }, + &cli.Command{ + Name: "gen-profiles", + Usage: "creates profile records for accounts", + Action: genProfiles, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "catalog", + Usage: "file path of account catalog JSON file", + Value: "data/fakermaker/accounts.json", + }, + &cli.BoolFlag{ + Name: "no-avatars", + Usage: "disable avatar image generation", + Value: false, + }, + &cli.BoolFlag{ + Name: "no-banners", + Usage: "disable profile banner image generation", + Value: false, + }, + }, + }, + &cli.Command{ + Name: "gen-graph", + Usage: "creates social graph (follows and mutes)", + Action: genGraph, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "catalog", + Usage: "file path of account catalog JSON file", + Value: "data/fakermaker/accounts.json", + }, + &cli.IntFlag{ + Name: "max-follows", + Usage: "create up to this many follows for each account", + Value: 90, + }, + &cli.IntFlag{ + Name: "max-mutes", + Usage: "create up to this many mutes (blocks) for each account", + Value: 25, + }, + }, + }, + &cli.Command{ + Name: "gen-posts", + Usage: "creates posts for accounts", + Action: genPosts, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "catalog", + Usage: "file path of account catalog JSON file", + Value: "data/fakermaker/accounts.json", + }, + &cli.IntFlag{ + Name: "max-posts", + Usage: "create up to this many posts for each account", + Value: 10, + }, + &cli.Float64Flag{ + Name: "frac-image", + Usage: "portion of posts to include images", + Value: 0.25, + }, + &cli.Float64Flag{ + Name: "frac-mention", + Usage: "of posts created, fraction to include mentions in", + Value: 0.50, + }, + }, + }, + &cli.Command{ + Name: "gen-interactions", + Usage: "create interactions between accounts", + Action: genInteractions, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "catalog", + Usage: "file path of account catalog JSON file", + Value: "data/fakermaker/accounts.json", + }, + &cli.Float64Flag{ + Name: "frac-like", + Usage: "fraction of posts in timeline to like", + Value: 0.20, + }, + &cli.Float64Flag{ + Name: "frac-repost", + Usage: "fraction of posts in timeline to repost", + Value: 0.20, + }, + &cli.Float64Flag{ + Name: "frac-reply", + Usage: "fraction of posts in timeline to reply to", + Value: 0.20, + }, + }, + }, + &cli.Command{ + Name: "run-browsing", + Usage: "creates read-only load on service (notifications, timeline, etc)", + Action: runBrowsing, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "catalog", + Usage: "file path of account catalog JSON file", + Value: "data/fakermaker/accounts.json", + }, + }, + }, + } + all := fakedata.MeasureIterations("entire command") + if err := app.Run(context.Background(), os.Args); err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + os.Exit(1) + } + all(1) +} + +// registers fake accounts with PDS, and spits out JSON-lines to stdout with auth info +func genAccounts(ctx context.Context, cmd *cli.Command) error { + + // establish atproto client, with admin token for auth + xrpcc, err := getXrpcClient(cmd, false) + if err != nil { + return err + } + adminToken := cmd.String("admin-password") + if len(adminToken) > 0 { + xrpcc.AdminToken = &adminToken + } + + countTotal := cmd.Int("count") + countCelebrities := cmd.Int("count-celebrities") + domainSuffix := cmd.String("domain-suffix") + if countCelebrities > countTotal { + return fmt.Errorf("more celebrities than total accounts!") + } + countRegulars := countTotal - countCelebrities + + var inviteCode *string = nil + if cmd.Bool("use-invite-code") { + resp, err := comatproto.ServerCreateInviteCodes(context.TODO(), xrpcc, &comatproto.ServerCreateInviteCodes_Input{ + UseCount: int64(countTotal), + ForAccounts: nil, + CodeCount: 1, + }) + if err != nil { + return err + } + if len(resp.Codes) != 1 || len(resp.Codes[0].Codes) != 1 { + return fmt.Errorf("expected a single invite code") + } + inviteCode = &resp.Codes[0].Codes[0] + } + + // call helper to do actual creation + var usr *fakedata.AccountContext + var line []byte + t1 := fakedata.MeasureIterations("register celebrity accounts") + for i := 0; i < countCelebrities; i++ { + if usr, err = fakedata.GenAccount(xrpcc, i, "celebrity", domainSuffix, inviteCode); err != nil { + return err + } + // compact single-line JSON by default + if line, err = json.Marshal(usr); err != nil { + return nil + } + fmt.Println(string(line)) + } + t1(countCelebrities) + + t2 := fakedata.MeasureIterations("register regular accounts") + for i := 0; i < countRegulars; i++ { + if usr, err = fakedata.GenAccount(xrpcc, i, "regular", domainSuffix, inviteCode); err != nil { + return err + } + // compact single-line JSON by default + if line, err = json.Marshal(usr); err != nil { + return nil + } + fmt.Println(string(line)) + } + t2(countRegulars) + return nil +} + +func genProfiles(ctx context.Context, cmd *cli.Command) error { + catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog")) + if err != nil { + return err + } + + pdsHost := cmd.String("pds-host") + genAvatar := !cmd.Bool("no-avatars") + genBanner := !cmd.Bool("no-banners") + jobs := cmd.Int("jobs") + + accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars)) + eg := new(errgroup.Group) + for i := 0; i < jobs; i++ { + eg.Go(func() error { + for acc := range accChan { + xrpcc, err := fakedata.AccountXrpcClient(pdsHost, &acc) + if err != nil { + return err + } + if err = fakedata.GenProfile(xrpcc, &acc, genAvatar, genBanner); err != nil { + return err + } + } + return nil + }) + } + + for _, acc := range append(catalog.Celebs, catalog.Regulars...) { + accChan <- acc + } + close(accChan) + return eg.Wait() +} + +func genGraph(ctx context.Context, cmd *cli.Command) error { + catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog")) + if err != nil { + return err + } + + pdsHost := cmd.String("pds-host") + maxFollows := cmd.Int("max-follows") + maxMutes := cmd.Int("max-mutes") + jobs := cmd.Int("jobs") + + accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars)) + eg := new(errgroup.Group) + for i := 0; i < jobs; i++ { + eg.Go(func() error { + for acc := range accChan { + xrpcc, err := fakedata.AccountXrpcClient(pdsHost, &acc) + if err != nil { + return err + } + if err = fakedata.GenFollowsAndMutes(xrpcc, catalog, &acc, maxFollows, maxMutes); err != nil { + return err + } + } + return nil + }) + } + + for _, acc := range append(catalog.Celebs, catalog.Regulars...) { + accChan <- acc + } + close(accChan) + return eg.Wait() +} + +func genPosts(ctx context.Context, cmd *cli.Command) error { + catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog")) + if err != nil { + return err + } + + pdsHost := cmd.String("pds-host") + maxPosts := cmd.Int("max-posts") + fracImage := cmd.Float64("frac-image") + fracMention := cmd.Float64("frac-mention") + jobs := cmd.Int("jobs") + + accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars)) + eg := new(errgroup.Group) + for i := 0; i < jobs; i++ { + eg.Go(func() error { + for acc := range accChan { + xrpcc, err := fakedata.AccountXrpcClient(pdsHost, &acc) + if err != nil { + return err + } + if err = fakedata.GenPosts(xrpcc, catalog, &acc, maxPosts, fracImage, fracMention); err != nil { + return err + } + } + return nil + }) + } + + for _, acc := range append(catalog.Celebs, catalog.Regulars...) { + accChan <- acc + } + close(accChan) + return eg.Wait() +} + +func genInteractions(ctx context.Context, cmd *cli.Command) error { + catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog")) + if err != nil { + return err + } + + pdsHost := cmd.String("pds-host") + fracLike := cmd.Float64("frac-like") + fracRepost := cmd.Float64("frac-repost") + fracReply := cmd.Float64("frac-reply") + jobs := cmd.Int("jobs") + + accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars)) + eg := new(errgroup.Group) + for i := 0; i < jobs; i++ { + eg.Go(func() error { + for acc := range accChan { + xrpcc, err := fakedata.AccountXrpcClient(pdsHost, &acc) + if err != nil { + return err + } + t1 := fakedata.MeasureIterations("all interactions") + if err := fakedata.GenLikesRepostsReplies(xrpcc, &acc, fracLike, fracRepost, fracReply); err != nil { + return err + } + t1(1) + } + return nil + }) + } + + for _, acc := range append(catalog.Celebs, catalog.Regulars...) { + accChan <- acc + } + close(accChan) + return eg.Wait() +} + +func runBrowsing(ctx context.Context, cmd *cli.Command) error { + catalog, err := fakedata.ReadAccountCatalog(cmd.String("catalog")) + if err != nil { + return err + } + + pdsHost := cmd.String("pds-host") + jobs := cmd.Int("jobs") + + accChan := make(chan fakedata.AccountContext, len(catalog.Celebs)+len(catalog.Regulars)) + eg := new(errgroup.Group) + for i := 0; i < jobs; i++ { + eg.Go(func() error { + for acc := range accChan { + xrpcc, err := fakedata.AccountXrpcClient(pdsHost, &acc) + if err != nil { + return err + } + if err := fakedata.BrowseAccount(xrpcc, &acc); err != nil { + return err + } + } + return nil + }) + } + + for _, acc := range append(catalog.Celebs, catalog.Regulars...) { + accChan <- acc + } + close(accChan) + return eg.Wait() +} + +func getXrpcClient(cmd *cli.Command, authreq bool) (*xrpc.Client, error) { + h := "http://localhost:4989" + if pdsurl := cmd.String("pds-host"); pdsurl != "" { + h = pdsurl + } + + auth, err := loadAuthFromEnv(cmd, authreq) + if err != nil { + return nil, fmt.Errorf("loading auth: %w", err) + } + + return &xrpc.Client{ + Client: cliutil.NewHttpClient(), + Host: h, + Auth: auth, + }, nil +} + +func loadAuthFromEnv(cmd *cli.Command, req bool) (*xrpc.AuthInfo, error) { + if a := cmd.String("auth"); a != "" { + if ai, err := cliutil.ReadAuth(a); err != nil && req { + return nil, err + } else { + return ai, nil + } + } + + val := os.Getenv("ATP_AUTH_FILE") + if val == "" { + if req { + return nil, fmt.Errorf("no auth env present, ATP_AUTH_FILE not set") + } + + return nil, nil + } + + var auth xrpc.AuthInfo + if err := json.Unmarshal([]byte(val), &auth); err != nil { + return nil, err + } + + return &auth, nil +} diff --git a/cmd/goat/README.md b/cmd/goat/README.md new file mode 100644 index 000000000..bcf637746 --- /dev/null +++ b/cmd/goat/README.md @@ -0,0 +1,4 @@ +`goat`: Go AT protocol CLI tool +=============================== + +**NOTE: this project has been moved to a dedicated git repo at [bluesky-social/goat](https://github.com/bluesky-social/goat)** diff --git a/cmd/gosky/account.go b/cmd/gosky/account.go new file mode 100644 index 000000000..6df0709ab --- /dev/null +++ b/cmd/gosky/account.go @@ -0,0 +1,217 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/util/cliutil" + + "github.com/urfave/cli/v2" +) + +var accountCmd = &cli.Command{ + Name: "account", + Usage: "sub-commands for auth session and account management", + Subcommands: []*cli.Command{ + createSessionCmd, + newAccountCmd, + refreshAuthTokenCmd, + resetPasswordCmd, + requestAccountDeletionCmd, + deleteAccountCmd, + }, +} + +var createSessionCmd = &cli.Command{ + Name: "create-session", + ArgsUsage: ` `, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + args, err := needArgs(cctx, "handle", "password") + if err != nil { + return err + } + handle, password := args[0], args[1] + + ses, err := comatproto.ServerCreateSession(context.TODO(), xrpcc, &comatproto.ServerCreateSession_Input{ + Identifier: handle, + Password: password, + }) + if err != nil { + return err + } + + b, err := json.MarshalIndent(ses, "", " ") + if err != nil { + return err + } + + fmt.Println(string(b)) + return nil + }, +} + +var newAccountCmd = &cli.Command{ + Name: "new", + ArgsUsage: ` [inviteCode]`, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + args, err := needArgs(cctx, "email", "handle", "password") + if err != nil { + return err + } + email, handle, password := args[0], args[1], args[2] + + var invite *string + if inv := cctx.Args().Get(3); inv != "" { + invite = &inv + } + + acc, err := comatproto.ServerCreateAccount(context.TODO(), xrpcc, &comatproto.ServerCreateAccount_Input{ + Email: &email, + Handle: handle, + InviteCode: invite, + Password: &password, + }) + if err != nil { + return err + } + + b, err := json.MarshalIndent(acc, "", " ") + if err != nil { + return err + } + + fmt.Println(string(b)) + return nil + }, +} + +var resetPasswordCmd = &cli.Command{ + Name: "reset-password", + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + ctx := context.TODO() + + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + args, err := needArgs(cctx, "email") + if err != nil { + return err + } + email := args[0] + + err = comatproto.ServerRequestPasswordReset(ctx, xrpcc, &comatproto.ServerRequestPasswordReset_Input{ + Email: email, + }) + if err != nil { + return err + } + + inp := bufio.NewScanner(os.Stdin) + fmt.Println("Enter recovery code from email:") + inp.Scan() + code := inp.Text() + + fmt.Println("Enter new password:") + inp.Scan() + npass := inp.Text() + + if err := comatproto.ServerResetPassword(ctx, xrpcc, &comatproto.ServerResetPassword_Input{ + Password: npass, + Token: code, + }); err != nil { + return err + } + + return nil + }, +} + +var refreshAuthTokenCmd = &cli.Command{ + Name: "refresh-session", + Usage: "refresh your auth token and overwrite it with new auth info", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + a := xrpcc.Auth + a.AccessJwt = a.RefreshJwt + + ctx := context.TODO() + nauth, err := comatproto.ServerRefreshSession(ctx, xrpcc) + if err != nil { + return err + } + + b, err := json.Marshal(nauth) + if err != nil { + return err + } + + if err := os.WriteFile(cctx.String("auth"), b, 0600); err != nil { + return err + } + + return nil + }, +} + +var requestAccountDeletionCmd = &cli.Command{ + Name: "request-deletion", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + err = comatproto.ServerRequestAccountDelete(cctx.Context, xrpcc) + if err != nil { + return err + } + + return nil + }, +} + +var deleteAccountCmd = &cli.Command{ + Name: "delete", + Usage: "permanently delete account", + ArgsUsage: " ", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + token := cctx.Args().First() + password := cctx.Args().Get(1) + + err = comatproto.ServerDeleteAccount(cctx.Context, xrpcc, &comatproto.ServerDeleteAccount_Input{ + Did: xrpcc.Auth.Did, + Token: token, + Password: password, + }) + if err != nil { + return err + } + + return nil + }, +} diff --git a/cmd/gosky/admin.go b/cmd/gosky/admin.go new file mode 100644 index 000000000..b35ca17f2 --- /dev/null +++ b/cmd/gosky/admin.go @@ -0,0 +1,851 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "sync" + "time" + + "github.com/bluesky-social/indigo/api/atproto" + comatproto "github.com/bluesky-social/indigo/api/atproto" + toolsozone "github.com/bluesky-social/indigo/api/ozone" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/handles" + "github.com/bluesky-social/indigo/util/cliutil" + + "github.com/urfave/cli/v2" +) + +var adminCmd = &cli.Command{ + Name: "admin", + Usage: "sub-commands for PDS administration", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "admin-password", + EnvVars: []string{"ATP_AUTH_ADMIN_PASSWORD"}, + Required: true, + }, + &cli.StringFlag{ + Name: "admin-endpoint", + Value: "https://mod.bsky.app", + }, + }, + Subcommands: []*cli.Command{ + buildInviteTreeCmd, + checkUserCmd, + createInviteCmd, + disableInvitesCmd, + enableInvitesCmd, + queryModerationStatusesCmd, + listInviteTreeCmd, + reportsCmd, + takeDownAccountCmd, + }, +} + +var checkUserCmd = &cli.Command{ + Name: "check-user", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "raw", + Usage: "dump simple JSON response to stdout", + }, + &cli.BoolFlag{ + Name: "list-invited-dids", + }, + }, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + dir := identity.DefaultDirectory() + ctx := context.Background() + + ident, err := syntax.ParseAtIdentifier(cctx.Args().First()) + if err != nil { + return err + } + + id, err := dir.Lookup(ctx, ident) + if err != nil { + return fmt.Errorf("resolve identifier %q: %w", cctx.Args().First(), err) + } + + did := id.DID.String() + + adminKey := cctx.String("admin-password") + xrpcc.AdminToken = &adminKey + xrpcc.Host = cctx.String("admin-endpoint") + + rep, err := toolsozone.ModerationGetRepo(ctx, xrpcc, did) + if err != nil { + return fmt.Errorf("getRepo %s: %w", did, err) + } + + b, err := json.MarshalIndent(rep, "", " ") + if err != nil { + return err + } + + if cctx.Bool("raw") { + fmt.Println(string(b)) + } else if cctx.Bool("list-invited-dids") { + for _, inv := range rep.Invites { + for _, u := range inv.Uses { + fmt.Println(u.UsedBy) + } + } + } else { + var invby string + if rep.InvitedBy != nil { + if fa := rep.InvitedBy.ForAccount; fa != "" { + if fa == "admin" { + invby = fa + } else { + id, err := dir.LookupDID(ctx, syntax.DID(fa)) + if err != nil { + fmt.Println("ERROR: failed to resolve inviter: ", err) + } + + invby = id.Handle.String() + } + } + } + + fmt.Println(rep.Handle) + fmt.Println(rep.Did) + if rep.Email != nil { + fmt.Println(*rep.Email) + } + fmt.Println("indexed at: ", rep.IndexedAt) + fmt.Printf("Invited by: %s\n", invby) + if rep.InvitesDisabled != nil && *rep.InvitesDisabled { + fmt.Println("INVITES DISABLED") + } + + var invited []*toolsozone.ModerationDefs_RepoViewDetail + var lk sync.Mutex + var wg sync.WaitGroup + var used int + var revoked int + for _, inv := range rep.Invites { + used += len(inv.Uses) + + if inv.Disabled { + revoked++ + } + for _, u := range inv.Uses { + wg.Add(1) + go func(did string) { + defer wg.Done() + repo, err := toolsozone.ModerationGetRepo(ctx, xrpcc, did) + if err != nil { + fmt.Println("ERROR: ", err) + return + } + + lk.Lock() + invited = append(invited, repo) + lk.Unlock() + }(u.UsedBy) + } + } + + wg.Wait() + + fmt.Printf("Invites, used %d of %d (%d disabled)\n", used, len(rep.Invites), revoked) + for _, inv := range invited { + + var invited, total int + for _, code := range inv.Invites { + total += len(code.Uses) + int(code.Available) + invited += len(code.Uses) + } + + fmt.Printf(" - %s (%d / %d)\n", inv.Handle, invited, total) + } + } + return nil + }, +} + +var buildInviteTreeCmd = &cli.Command{ + Name: "build-invite-tree", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "invite-list", + }, + &cli.IntFlag{ + Name: "top", + Value: 50, + }, + }, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.Background() + + adminKey := cctx.String("admin-password") + + xrpcc.AdminToken = &adminKey + + var allcodes []*atproto.ServerDefs_InviteCode + + if invl := cctx.String("invite-list"); invl != "" { + fi, err := os.Open(invl) + if err != nil { + return err + } + + if err := json.NewDecoder(fi).Decode(&allcodes); err != nil { + return err + } + } else { + var cursor string + for { + invites, err := atproto.AdminGetInviteCodes(ctx, xrpcc, cursor, 100, "") + if err != nil { + return err + } + + allcodes = append(allcodes, invites.Codes...) + + if invites.Cursor != nil { + cursor = *invites.Cursor + } + if len(invites.Codes) == 0 { + break + } + } + + fi, err := os.Create("output.json") + if err != nil { + return err + } + defer fi.Close() + + if err := json.NewEncoder(fi).Encode(allcodes); err != nil { + return err + } + } + + users := make(map[string]*userInviteInfo) + users["admin"] = &userInviteInfo{ + Handle: "admin", + } + + var getUser func(did string) (*userInviteInfo, error) + getUser = func(did string) (*userInviteInfo, error) { + u, ok := users[did] + if ok { + return u, nil + } + + repo, err := toolsozone.ModerationGetRepo(ctx, xrpcc, did) + if err != nil { + return nil, err + } + + var invby string + if fa := repo.InvitedBy.ForAccount; fa != "" { + if fa == "admin" { + invby = "admin" + } else { + invu, ok := users[fa] + if ok { + invby = invu.Handle + } else { + invrepo, err := toolsozone.ModerationGetRepo(ctx, xrpcc, fa) + if err != nil { + return nil, fmt.Errorf("resolving inviter (%q): %w", fa, err) + } + + invby = invrepo.Handle + } + } + } + + u = &userInviteInfo{ + Did: did, + Handle: repo.Handle, + InvitedBy: repo.InvitedBy.ForAccount, + InvitedByHandle: invby, + TotalInvites: len(repo.Invites), + } + if repo.Email != nil { + u.Email = *repo.Email + } + + users[did] = u + + return u, nil + } + _ = getUser + + initmap := make(map[string]*basicInvInfo) + var initlist []*basicInvInfo + for _, inv := range allcodes { + acc, ok := initmap[inv.ForAccount] + if !ok { + acc = &basicInvInfo{ + Did: inv.ForAccount, + } + initmap[inv.ForAccount] = acc + initlist = append(initlist, acc) + } + + acc.TotalInvites += int(inv.Available) + len(inv.Uses) + for _, u := range inv.Uses { + acc.Invited = append(acc.Invited, u.UsedBy) + } + } + + sort.Slice(initlist, func(i, j int) bool { + return len(initlist[i].Invited) > len(initlist[j].Invited) + }) + + for i := 0; i < cctx.Int("top"); i++ { + u, err := getUser(initlist[i].Did) + if err != nil { + fmt.Printf("getuser %q: %s\n", initlist[i].Did, err) + continue + } + + fmt.Printf("%d: %s (%d of %d)\n", i, u.Handle, len(initlist[i].Invited), u.TotalInvites) + } + + /* + fmt.Println("writing output...") + outfi, err := os.Create("userdump.json") + if err != nil { + return err + } + defer outfi.Close() + + return json.NewEncoder(outfi).Encode(users) + */ + + return nil + }, +} + +type userInviteInfo struct { + CreatedAt time.Time + Did string + Handle string + InvitedBy string + InvitedByHandle string + TotalInvites int + Invited []string + Email string +} + +type basicInvInfo struct { + Did string + Invited []string + TotalInvites int +} + +var reportsCmd = &cli.Command{ + Name: "reports", + Subcommands: []*cli.Command{ + listReportsCmd, + }, +} + +var listReportsCmd = &cli.Command{ + Name: "list", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "raw", + }, + &cli.BoolFlag{ + Name: "resolved", + Value: true, + }, + &cli.BoolFlag{ + Name: "template-output", + }, + }, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.Background() + + adminKey := cctx.String("admin-password") + xrpcc.AdminToken = &adminKey + + // fetch recent moderation reports + resp, err := toolsozone.ModerationQueryEvents( + ctx, + xrpcc, + nil, // addedLabels []string + nil, // addedTags []string + "", // ageAssuranceState + "", // batchId string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + "", // createdBy string + "", // cursor string + false, // hasComment bool + false, // includeAllUserRecords bool + 100, // limit int64 + nil, // modTool + nil, // policies []string + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + "", // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string + false, // withStrike bool + ) + if err != nil { + return err + } + + for _, rep := range resp.Events { + b, err := json.MarshalIndent(rep, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + } + return nil + }, +} + +var disableInvitesCmd = &cli.Command{ + Name: "disable-invites", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.Background() + + adminKey := cctx.String("admin-password") + xrpcc.AdminToken = &adminKey + + phr := &handles.ProdHandleResolver{} + handle := cctx.Args().First() + if !strings.HasPrefix(handle, "did:") { + resp, err := phr.ResolveHandleToDid(ctx, handle) + if err != nil { + return err + } + + handle = resp + } + + if err := atproto.AdminDisableAccountInvites(ctx, xrpcc, &atproto.AdminDisableAccountInvites_Input{ + Account: handle, + }); err != nil { + return err + } + + if err := atproto.AdminDisableInviteCodes(ctx, xrpcc, &atproto.AdminDisableInviteCodes_Input{ + Accounts: []string{handle}, + }); err != nil { + return err + } + + return nil + }, +} + +var enableInvitesCmd = &cli.Command{ + Name: "enable-invites", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.Background() + + adminKey := cctx.String("admin-password") + xrpcc.AdminToken = &adminKey + + handle := cctx.Args().First() + if !strings.HasPrefix(handle, "did:") { + phr := &handles.ProdHandleResolver{} + resp, err := phr.ResolveHandleToDid(ctx, handle) + if err != nil { + return err + } + + handle = resp + } + + return atproto.AdminEnableAccountInvites(ctx, xrpcc, &atproto.AdminEnableAccountInvites_Input{ + Account: handle, + }) + }, +} + +var listInviteTreeCmd = &cli.Command{ + Name: "list-invite-tree", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "disable-invites", + Usage: "additionally disable invites for all printed DIDs", + }, + &cli.BoolFlag{ + Name: "revoke-existing-invites", + Usage: "additionally revoke any existing invites for all printed DIDs", + }, + &cli.BoolFlag{ + Name: "print-handles", + Usage: "print handle for each DID", + }, + &cli.BoolFlag{ + Name: "print-emails", + Usage: "print account email for each DID", + }, + }, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.Background() + + phr := &handles.ProdHandleResolver{} + + did := cctx.Args().First() + if !strings.HasPrefix(did, "did:") { + rdid, err := phr.ResolveHandleToDid(ctx, cctx.Args().First()) + if err != nil { + return fmt.Errorf("resolve handle %q: %w", cctx.Args().First(), err) + } + + did = rdid + } + + adminKey := cctx.String("admin-password") + xrpcc.AdminToken = &adminKey + + queue := []string{did} + + for len(queue) > 0 { + next := queue[0] + queue = queue[1:] + + if cctx.Bool("disable-invites") { + if err := atproto.AdminDisableAccountInvites(ctx, xrpcc, &atproto.AdminDisableAccountInvites_Input{ + Account: next, + }); err != nil { + return fmt.Errorf("failed to disable invites on %q: %w", next, err) + } + } + + if cctx.Bool("revoke-existing-invites") { + if err := atproto.AdminDisableInviteCodes(ctx, xrpcc, &atproto.AdminDisableInviteCodes_Input{ + Accounts: []string{next}, + }); err != nil { + return fmt.Errorf("failed to revoke existing invites on %q: %w", next, err) + } + } + + rep, err := toolsozone.ModerationGetRepo(ctx, xrpcc, next) + if err != nil { + fmt.Printf("Failed to getRepo for DID %s: %s\n", next, err.Error()) + continue + } + fmt.Print(next) + + if cctx.Bool("print-handles") { + if rep.Handle != "" { + fmt.Print(" ", rep.Handle) + } else { + fmt.Print(" NO HANDLE") + } + } + + if cctx.Bool("print-emails") { + if rep.Email != nil { + fmt.Print(" ", *rep.Email) + } else { + fmt.Print(" NO EMAIL") + } + } + fmt.Println() + + for _, inv := range rep.Invites { + for _, u := range inv.Uses { + queue = append(queue, u.UsedBy) + } + } + } + return nil + }, +} + +var takeDownAccountCmd = &cli.Command{ + Name: "account-takedown", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "reason", + Usage: "why the account is being taken down", + Required: true, + }, + &cli.StringFlag{ + Name: "admin-user", + Usage: "account of person running this command, for recordkeeping", + Required: true, + }, + }, + Action: func(cctx *cli.Context) error { + + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.Background() + + adminKey := cctx.String("admin-password") + xrpcc.AdminToken = &adminKey + + for _, did := range cctx.Args().Slice() { + if !strings.HasPrefix(did, "did:") { + dir := identity.DefaultDirectory() + resp, err := dir.LookupHandle(ctx, syntax.Handle(did)) + if err != nil { + return err + } + + did = resp.DID.String() + } + + reason := cctx.String("reason") + adminUser := cctx.String("admin-user") + if !strings.HasPrefix(adminUser, "did:") { + dir := identity.DefaultDirectory() + resp, err := dir.LookupHandle(ctx, syntax.Handle(adminUser)) + if err != nil { + return err + } + + adminUser = resp.DID.String() + } + + resp, err := toolsozone.ModerationEmitEvent(ctx, xrpcc, &toolsozone.ModerationEmitEvent_Input{ + CreatedBy: adminUser, + Event: &toolsozone.ModerationEmitEvent_Input_Event{ + ModerationDefs_ModEventTakedown: &toolsozone.ModerationDefs_ModEventTakedown{ + Comment: &reason, + }, + }, + Subject: &toolsozone.ModerationEmitEvent_Input_Subject{ + AdminDefs_RepoRef: &atproto.AdminDefs_RepoRef{ + Did: did, + }, + }, + }) + if err != nil { + return err + } + + b, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return err + } + + fmt.Println(string(b)) + } + return nil + }, +} + +var queryModerationStatusesCmd = &cli.Command{ + Name: "query-moderation-statuses", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.Background() + + adminKey := cctx.String("admin-password") + xrpcc.AdminToken = &adminKey + + did := cctx.Args().First() + if !strings.HasPrefix(did, "did:") { + phr := &handles.ProdHandleResolver{} + resp, err := phr.ResolveHandleToDid(ctx, did) + if err != nil { + return err + } + + did = resp + } + + resp, err := toolsozone.ModerationQueryEvents( + ctx, + xrpcc, + nil, // addedLabels []string + nil, // addedTags []string + "", // ageAssuranceState + "", // batchId string + nil, // collections []string + "", // comment string + "", // createdAfter string + "", // createdBefore string + "", // createdBy string + "", // cursor string + false, // hasComment bool + false, // includeAllUserRecords bool + 100, // limit int64 + nil, // modTool + nil, // policies []string + nil, // removedLabels []string + nil, // removedTags []string + nil, // reportTypes []string + "", // sortDirection string + "", // subject string + "", // subjectType string + []string{"tools.ozone.moderation.defs#modEventReport"}, // types []string + false, // withStrike bool + ) + if err != nil { + return err + } + + b, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return err + } + + fmt.Println(string(b)) + return nil + }, +} + +var createInviteCmd = &cli.Command{ + Name: "create-invites", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "useCount", + Value: 1, + }, + &cli.IntFlag{ + Name: "num", + Value: 1, + }, + &cli.StringFlag{ + Name: "bulk", + }, + }, + ArgsUsage: "[handle]", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + adminKey := cctx.String("admin-password") + + count := cctx.Int("useCount") + num := cctx.Int("num") + + phr := &handles.ProdHandleResolver{} + if bulkfi := cctx.String("bulk"); bulkfi != "" { + xrpcc.AdminToken = &adminKey + dids, err := readDids(bulkfi) + if err != nil { + return err + } + + for i, d := range dids { + if !strings.HasPrefix(d, "did:plc:") { + out, err := phr.ResolveHandleToDid(context.TODO(), d) + if err != nil { + return fmt.Errorf("failed to resolve %q: %w", d, err) + } + + dids[i] = out + } + } + + for n := 0; n < len(dids); n += 500 { + slice := dids + if len(slice) > 500 { + slice = slice[:500] + } + + _, err = comatproto.ServerCreateInviteCodes(context.TODO(), xrpcc, &comatproto.ServerCreateInviteCodes_Input{ + UseCount: int64(count), + ForAccounts: slice, + CodeCount: int64(num), + }) + if err != nil { + return err + } + } + + return nil + } + + var usrdid []string + if forUser := cctx.Args().Get(0); forUser != "" { + if !strings.HasPrefix(forUser, "did:") { + resp, err := phr.ResolveHandleToDid(context.TODO(), forUser) + if err != nil { + return fmt.Errorf("resolving handle: %w", err) + } + + usrdid = []string{resp} + } else { + usrdid = []string{forUser} + } + } + + xrpcc.AdminToken = &adminKey + + resp, err := comatproto.ServerCreateInviteCodes(context.TODO(), xrpcc, &comatproto.ServerCreateInviteCodes_Input{ + UseCount: int64(count), + ForAccounts: usrdid, + CodeCount: int64(num), + }) + if err != nil { + return fmt.Errorf("creating codes: %w", err) + } + + for _, c := range resp.Codes { + for _, cc := range c.Codes { + fmt.Println(cc) + } + } + + return nil + }, +} diff --git a/cmd/gosky/bgs.go b/cmd/gosky/bgs.go new file mode 100644 index 000000000..20867f5ca --- /dev/null +++ b/cmd/gosky/bgs.go @@ -0,0 +1,527 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/bluesky-social/indigo/xrpc" + + "github.com/urfave/cli/v2" +) + +var bgsAdminCmd = &cli.Command{ + Name: "bgs", + Usage: "sub-commands for administering a BGS", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "key", + EnvVars: []string{"BGS_ADMIN_KEY"}, + }, + &cli.StringFlag{ + Name: "bgs", + Value: "http://localhost:2470", + }, + }, + Subcommands: []*cli.Command{ + bgsListUpstreamsCmd, + bgsKickConnectionCmd, + bgsListDomainBansCmd, + bgsBanDomainCmd, + bgsTakedownRepoCmd, + bgsSetNewSubsEnabledCmd, + bgsCompactRepo, + bgsCompactAll, + bgsResetRepo, + }, +} + +var bgsListUpstreamsCmd = &cli.Command{ + Name: "list", + Action: func(cctx *cli.Context) error { + url := cctx.String("bgs") + "/admin/subs/getUpstreamConns" + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out []string + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + for _, h := range out { + fmt.Println(h) + } + + return nil + }, +} + +var bgsKickConnectionCmd = &cli.Command{ + Name: "kick", + Usage: "tell Relay/BGS to drop the subscription connection", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "ban", + Usage: "make the disconnect sticky", + }, + }, + Action: func(cctx *cli.Context) error { + uu := cctx.String("bgs") + "/admin/subs/killUpstream?host=" + + uu += url.QueryEscape(cctx.Args().First()) + + if cctx.Bool("ban") { + uu += "&block=true" + } + + req, err := http.NewRequest("POST", uu, nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} + +var bgsListDomainBansCmd = &cli.Command{ + Name: "list-domain-bans", + Action: func(cctx *cli.Context) error { + url := cctx.String("bgs") + "/admin/subs/listDomainBans" + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out []string + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + for _, h := range out { + fmt.Println(h) + } + + return nil + }, +} + +var bgsBanDomainCmd = &cli.Command{ + Name: "ban-domain", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + url := cctx.String("bgs") + "/admin/subs/banDomain" + + b, err := json.Marshal(map[string]string{ + "domain": cctx.Args().First(), + }) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", url, bytes.NewReader(b)) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} + +var bgsTakedownRepoCmd = &cli.Command{ + Name: "take-down-repo", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + url := cctx.String("bgs") + "/admin/repo/takeDown" + + b, err := json.Marshal(map[string]string{ + "did": cctx.Args().First(), + }) + if err != nil { + return err + } + + req, err := http.NewRequest("POST", url, bytes.NewReader(b)) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} + +var bgsSetNewSubsEnabledCmd = &cli.Command{ + Name: "set-accept-subs", + ArgsUsage: "", + Usage: "set configuration for whether new subscriptions are allowed", + Action: func(cctx *cli.Context) error { + url := cctx.String("bgs") + "/admin/subs/setEnabled" + + bv, err := strconv.ParseBool(cctx.Args().First()) + if err != nil { + return err + } + + url += fmt.Sprintf("?enabled=%v", bv) + + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} + +var bgsCompactRepo = &cli.Command{ + Name: "compact-repo", + ArgsUsage: "", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "fast", + }, + }, + Action: func(cctx *cli.Context) error { + uu, err := url.Parse(cctx.String("bgs") + "/admin/repo/compact") + if err != nil { + return err + } + + q := uu.Query() + did := cctx.Args().First() + q.Add("did", did) + + if cctx.Bool("fast") { + q.Add("fast", "true") + } + + uu.RawQuery = q.Encode() + + req, err := http.NewRequest("POST", uu.String(), nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} + +var bgsCompactAll = &cli.Command{ + Name: "compact-all", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "dry", + }, + &cli.IntFlag{ + Name: "limit", + }, + &cli.IntFlag{ + Name: "threshold", + }, + &cli.BoolFlag{ + Name: "fast", + }, + }, + Action: func(cctx *cli.Context) error { + uu, err := url.Parse(cctx.String("bgs") + "/admin/repo/compactAll") + if err != nil { + return err + } + + q := uu.Query() + if cctx.Bool("dry") { + q.Add("dry", "true") + } + + if cctx.Bool("fast") { + q.Add("fast", "true") + } + + if cctx.IsSet("limit") { + q.Add("limit", fmt.Sprint(cctx.Int("limit"))) + } + + if cctx.IsSet("threshold") { + q.Add("threshold", fmt.Sprint(cctx.Int("threshold"))) + } + + uu.RawQuery = q.Encode() + + req, err := http.NewRequest("POST", uu.String(), nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} + +var bgsResetRepo = &cli.Command{ + Name: "reset-repo", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + url := cctx.String("bgs") + "/admin/repo/reset" + + did := cctx.Args().First() + url += fmt.Sprintf("?did=%s", did) + + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} + +var bgsSetTrustedDomains = &cli.Command{ + Name: "set-trusted-domain", + Action: func(cctx *cli.Context) error { + url := cctx.String("bgs") + "/admin/pds/addTrustedDomain" + + domain := cctx.Args().First() + url += fmt.Sprintf("?domain=%s", domain) + + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} diff --git a/cmd/gosky/bsky.go b/cmd/gosky/bsky.go new file mode 100644 index 000000000..cc1641de7 --- /dev/null +++ b/cmd/gosky/bsky.go @@ -0,0 +1,354 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/util/cliutil" + + "github.com/urfave/cli/v2" +) + +var bskyCmd = &cli.Command{ + Name: "bsky", + Usage: "sub-commands for bsky-specific endpoints", + Subcommands: []*cli.Command{ + bskyFollowCmd, + bskyListFollowsCmd, + bskyPostCmd, + bskyGetFeedCmd, + bskyLikeCmd, + bskyDeletePostCmd, + bskyActorGetSuggestionsCmd, + bskyNotificationsCmd, + }, +} + +var bskyFollowCmd = &cli.Command{ + Name: "follow", + Usage: "create a follow relationship (auth required)", + Flags: []cli.Flag{}, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + user := cctx.Args().First() + + follow := appbsky.GraphFollow{ + LexiconTypeID: "app.bsky.graph.follow", + CreatedAt: time.Now().Format(time.RFC3339), + Subject: user, + } + + resp, err := comatproto.RepoCreateRecord(context.TODO(), xrpcc, &comatproto.RepoCreateRecord_Input{ + Collection: "app.bsky.graph.follow", + Repo: xrpcc.Auth.Did, + Record: &lexutil.LexiconTypeDecoder{Val: &follow}, + }) + if err != nil { + return err + } + + fmt.Println(resp.Uri) + + return nil + }, +} + +var bskyListFollowsCmd = &cli.Command{ + Name: "list-follows", + Usage: "print list of follows for account", + ArgsUsage: `[actor]`, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + user := cctx.Args().First() + if user == "" { + user = xrpcc.Auth.Did + } + + ctx := context.TODO() + resp, err := appbsky.GraphGetFollows(ctx, xrpcc, user, "", 100) + if err != nil { + return err + } + + for _, f := range resp.Follows { + fmt.Println(f.Did, f.Handle) + } + + return nil + }, +} + +var bskyPostCmd = &cli.Command{ + Name: "post", + Usage: "create a post record", + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + auth := xrpcc.Auth + + text := strings.Join(cctx.Args().Slice(), " ") + + resp, err := comatproto.RepoCreateRecord(context.TODO(), xrpcc, &comatproto.RepoCreateRecord_Input{ + Collection: "app.bsky.feed.post", + Repo: auth.Did, + Record: &lexutil.LexiconTypeDecoder{Val: &appbsky.FeedPost{ + Text: text, + CreatedAt: time.Now().Format(util.ISO8601), + }}, + }) + if err != nil { + return fmt.Errorf("failed to create post: %w", err) + } + + fmt.Println(resp.Cid) + fmt.Println(resp.Uri) + + return nil + }, +} + +func prettyPrintPost(p *appbsky.FeedDefs_FeedViewPost, uris bool) { + fmt.Println(strings.Repeat("-", 60)) + rec := p.Post.Record.Val.(*appbsky.FeedPost) + fmt.Printf("%s (%s)", p.Post.Author.Handle, rec.CreatedAt) + if uris { + fmt.Println(" -- ", p.Post.Uri) + } else { + fmt.Println(":") + } + fmt.Println(rec.Text) +} + +var bskyGetFeedCmd = &cli.Command{ + Name: "get-feed", + Usage: "fetch bsky feed", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "count", + Value: 100, + }, + &cli.StringFlag{ + Name: "author", + Usage: "specify handle of user to list their authored feed", + }, + &cli.BoolFlag{ + Name: "raw", + Usage: "print out feed in raw json", + }, + &cli.BoolFlag{ + Name: "uris", + Usage: "include URIs in pretty print output", + }, + }, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + ctx := context.TODO() + + raw := cctx.Bool("raw") + + uris := cctx.Bool("uris") + + author := cctx.String("author") + if author != "" { + if author == "self" { + author = xrpcc.Auth.Did + } + + tl, err := appbsky.FeedGetAuthorFeed(ctx, xrpcc, author, "", "", false, 99) + if err != nil { + return err + } + + for i := len(tl.Feed) - 1; i >= 0; i-- { + it := tl.Feed[i] + if raw { + jsonPrint(it) + } else { + prettyPrintPost(it, uris) + } + } + } else { + algo := "reverse-chronological" + tl, err := appbsky.FeedGetTimeline(ctx, xrpcc, algo, "", int64(cctx.Int("count"))) + if err != nil { + return err + } + + for i := len(tl.Feed) - 1; i >= 0; i-- { + it := tl.Feed[i] + if raw { + jsonPrint(it) + } else { + prettyPrintPost(it, uris) + } + } + } + + return nil + + }, +} + +var bskyActorGetSuggestionsCmd = &cli.Command{ + Name: "actor-get-suggestions", + ArgsUsage: "[author]", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + ctx := context.TODO() + + author := cctx.Args().First() + if author == "" { + author = xrpcc.Auth.Did + } + + resp, err := appbsky.ActorGetSuggestions(ctx, xrpcc, "", 100) + if err != nil { + return err + } + + b, err := json.MarshalIndent(resp.Actors, "", " ") + if err != nil { + return err + } + + fmt.Println(string(b)) + + return nil + + }, +} + +var bskyLikeCmd = &cli.Command{ + Name: "like", + Usage: "create bsky 'like' record", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + arg := cctx.Args().First() + + parts := strings.Split(arg, "/") + if len(parts) < 3 { + return fmt.Errorf("invalid post uri: %q", arg) + } + rkey := parts[len(parts)-1] + collection := parts[len(parts)-2] + did := parts[2] + + fmt.Println(did, collection, rkey) + ctx := context.TODO() + resp, err := comatproto.RepoGetRecord(ctx, xrpcc, "", collection, did, rkey) + if err != nil { + return fmt.Errorf("getting record: %w", err) + } + + out, err := comatproto.RepoCreateRecord(ctx, xrpcc, &comatproto.RepoCreateRecord_Input{ + Collection: "app.bsky.feed.like", + Repo: xrpcc.Auth.Did, + Record: &lexutil.LexiconTypeDecoder{ + Val: &appbsky.FeedLike{ + CreatedAt: time.Now().Format(util.ISO8601), + Subject: &comatproto.RepoStrongRef{Uri: resp.Uri, Cid: *resp.Cid}, + }, + }, + }) + if err != nil { + return fmt.Errorf("creating like failed: %w", err) + } + _ = out + return nil + + }, +} + +var bskyDeletePostCmd = &cli.Command{ + Name: "delete-post", + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + rkey := cctx.Args().First() + + if rkey == "" { + return fmt.Errorf("must specify rkey of post to delete") + } + + schema := "app.bsky.feed.post" + if strings.Contains(rkey, "/") { + parts := strings.Split(rkey, "/") + schema = parts[0] + rkey = parts[1] + } + + _, err = comatproto.RepoDeleteRecord(context.TODO(), xrpcc, &comatproto.RepoDeleteRecord_Input{ + Repo: xrpcc.Auth.Did, + Collection: schema, + Rkey: rkey, + }) + return err + }, +} + +var bskyNotificationsCmd = &cli.Command{ + Name: "notifs", + Usage: "fetch bsky notifications (requires auth)", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + ctx := context.TODO() + + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + notifs, err := appbsky.NotificationListNotifications(ctx, xrpcc, "", 50, false, nil, "") + if err != nil { + return err + } + + for _, n := range notifs.Notifications { + b, err := json.Marshal(n) + if err != nil { + return err + } + + fmt.Println(string(b)) + } + + return nil + }, +} diff --git a/cmd/gosky/car.go b/cmd/gosky/car.go new file mode 100644 index 000000000..2098a2cdf --- /dev/null +++ b/cmd/gosky/car.go @@ -0,0 +1,125 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/repo" + + "github.com/ipfs/go-cid" + "github.com/urfave/cli/v2" +) + +var carCmd = &cli.Command{ + Name: "car", + Usage: "sub-commands to work with CAR files on local disk", + Subcommands: []*cli.Command{ + carUnpackCmd, + }, +} + +var carUnpackCmd = &cli.Command{ + Name: "unpack", + Usage: "read all records from repo export CAR file, write as JSON files in directories", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "cbor", + Usage: "output CBOR files instead of JSON", + }, + &cli.StringFlag{ + Name: "out-dir", + Usage: "directory to write files to", + }, + }, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + arg := cctx.Args().First() + if arg == "" { + return fmt.Errorf("CAR file path arg is required") + } + + fi, err := os.Open(arg) + if err != nil { + return err + } + + r, err := repo.ReadRepoFromCar(ctx, fi) + if err != nil { + return err + } + + sc := r.SignedCommit() + did, err := syntax.ParseDID(sc.Did) + if err != nil { + return err + } + + topDir := cctx.String("out-dir") + if topDir == "" { + topDir = did.String() + } + log.Info("writing output", "topDir", topDir) + + commitPath := topDir + "/_commit" + os.MkdirAll(filepath.Dir(commitPath), os.ModePerm) + if cctx.Bool("cbor") { + cborBytes := new(bytes.Buffer) + err = sc.MarshalCBOR(cborBytes) + if err := os.WriteFile(commitPath+".cbor", cborBytes.Bytes(), 0666); err != nil { + return err + } + } else { + recJson, err := json.MarshalIndent(sc, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(commitPath+".json", recJson, 0666); err != nil { + return err + } + } + + err = r.ForEach(ctx, "", func(k string, v cid.Cid) error { + + _, rec, err := r.GetRecord(ctx, k) + if err != nil { + return err + } + log.Debug("processing record", "rec", k) + + // TODO: check if path is safe more carefully + recPath := topDir + "/" + k + os.MkdirAll(filepath.Dir(recPath), os.ModePerm) + if err != nil { + return err + } + if cctx.Bool("cbor") { + cborBytes := new(bytes.Buffer) + err = rec.MarshalCBOR(cborBytes) + if err := os.WriteFile(recPath+".cbor", cborBytes.Bytes(), 0666); err != nil { + return err + } + } else { + recJson, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(recPath+".json", recJson, 0666); err != nil { + return err + } + } + + return nil + }) + if err != nil { + return err + } + + return nil + }, +} diff --git a/cmd/gosky/debug.go b/cmd/gosky/debug.go new file mode 100644 index 000000000..46c7b3708 --- /dev/null +++ b/cmd/gosky/debug.go @@ -0,0 +1,989 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/bluesky-social/indigo/api/atproto" + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/did" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/schedulers/sequential" + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/repo" + "github.com/bluesky-social/indigo/repomgr" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/util/cliutil" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/gorilla/websocket" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipld/go-car/v2" + "github.com/urfave/cli/v2" +) + +var debugCmd = &cli.Command{ + Name: "debug", + Usage: "a set of debugging utilities for atproto", + Subcommands: []*cli.Command{ + inspectEventCmd, + debugStreamCmd, + debugFeedGenCmd, + debugFeedViewCmd, + compareStreamsCmd, + debugGetRepoCmd, + debugCompareReposCmd, + }, +} + +var inspectEventCmd = &cli.Command{ + Name: "inspect-event", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Required: true, + }, + &cli.BoolFlag{ + Name: "dump-raw-blocks", + }, + }, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + n, err := strconv.Atoi(cctx.Args().First()) + if err != nil { + return err + } + + h := cctx.String("host") + + url := fmt.Sprintf("%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", h, n-1) + d := websocket.DefaultDialer + con, _, err := d.Dial(url, http.Header{}) + if err != nil { + return fmt.Errorf("dial failure: %w", err) + } + + var errFoundIt = fmt.Errorf("gotem") + + var match *comatproto.SyncSubscribeRepos_Commit + + ctx := context.TODO() + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + n := int64(n) + if evt.Seq == n { + match = evt + return errFoundIt + } + if evt.Seq > n { + return fmt.Errorf("record not found in stream") + } + + return nil + }, + RepoInfo: func(evt *comatproto.SyncSubscribeRepos_Info) error { + return nil + }, + // TODO: all the other Repo* event types + Error: func(evt *events.ErrorFrame) error { + return fmt.Errorf("%s: %s", evt.Error, evt.Message) + }, + } + + seqScheduler := sequential.NewScheduler("debug-inspect-event", rsc.EventHandler) + err = events.HandleRepoStream(ctx, con, seqScheduler, nil) + if err != errFoundIt { + return err + } + + b, err := json.MarshalIndent(match, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + + br, err := car.NewBlockReader(bytes.NewReader(match.Blocks)) + if err != nil { + return err + } + + fmt.Println("\nSlice Dump:") + fmt.Println("Root: ", br.Roots[0]) + for { + blk, err := br.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + fmt.Println(blk.Cid()) + if cctx.Bool("dump-raw-blocks") { + fmt.Printf("%x\n", blk.RawData()) + } + } + + r, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(match.Blocks)) + if err != nil { + return fmt.Errorf("opening repo from slice: %w", err) + } + + fmt.Println("\nOps: ") + for _, op := range match.Ops { + switch repomgr.EventKind(op.Action) { + case repomgr.EvtKindCreateRecord, repomgr.EvtKindUpdateRecord: + rcid, _, err := r.GetRecord(ctx, op.Path) + if err != nil { + return fmt.Errorf("loading %q: %w", op.Path, err) + } + if rcid != cid.Cid(*op.Cid) { + return fmt.Errorf("mismatch in record cid %s != %s", rcid, *op.Cid) + } + fmt.Printf("%s (%s): %s\n", op.Action, op.Path, *op.Cid) + } + } + + return nil + }, +} + +type eventInfo struct { + LastSeq int64 + LastRev string +} + +func cidStr(c *lexutil.LexLink) string { + if c == nil { + return "" + } + + return c.String() +} + +var debugStreamCmd = &cli.Command{ + Name: "debug-stream", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + Required: true, + }, + &cli.BoolFlag{ + Name: "dump-raw-blocks", + }, + }, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + n, err := strconv.Atoi(cctx.Args().First()) + if err != nil { + return err + } + + h := cctx.String("host") + + url := fmt.Sprintf("%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", h, n) + d := websocket.DefaultDialer + con, _, err := d.Dial(url, http.Header{}) + if err != nil { + return fmt.Errorf("dial failure: %w", err) + } + + infos := make(map[string]*eventInfo) + + var lastSeq int64 = -1 + ctx := context.TODO() + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + + fmt.Printf("\rChecking seq: %d ", evt.Seq) + if lastSeq > 0 && evt.Seq != lastSeq+1 { + fmt.Println("Gap in sequence numbers: ", lastSeq, evt.Seq) + } + lastSeq = evt.Seq + + if !evt.TooBig { + r, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks)) + if err != nil { + fmt.Printf("\nEvent at sequence %d had an invalid repo slice: %s\n", evt.Seq, err) + return nil + } else { + _ = r + /* "prev" is no longer included in #commit messages + prev, err := r.PrevCommit(ctx) + if err != nil { + return err + } + + var cs, es string + if prev != nil { + cs = prev.String() + } + + if evt.Prev != nil { + es = evt.Prev.String() + } + + if !evt.Rebase && cs != es { + fmt.Printf("\nEvent at sequence %d has mismatch between slice prev and struct prev: %s != %s\n", evt.Seq, prev, evt.Prev) + } + */ + } + } + + cur, ok := infos[evt.Repo] + if ok { + if evt.Since != nil && cur.LastRev != *evt.Since { + /* + fmt.Println() + fmt.Printf("Event at sequence %d, repo=%s had since=%s, but last rev we saw was %s (seq=%d)\n", evt.Seq, evt.Repo, evt.Since, cur.LastRev, cur.LastSeq) + */ + } + } + + infos[evt.Repo] = &eventInfo{ + LastSeq: evt.Seq, + LastRev: evt.Rev, + } + + return nil + }, + RepoSync: func(evt *comatproto.SyncSubscribeRepos_Sync) error { + fmt.Printf("\rChecking seq: %d ", evt.Seq) + if lastSeq > 0 && evt.Seq != lastSeq+1 { + fmt.Println("Gap in sequence numbers: ", lastSeq, evt.Seq) + } + lastSeq = evt.Seq + return nil + }, + RepoInfo: func(evt *comatproto.SyncSubscribeRepos_Info) error { + return nil + }, + // TODO: all the other Repo* event types + Error: func(evt *events.ErrorFrame) error { + return fmt.Errorf("%s: %s", evt.Error, evt.Message) + }, + } + seqScheduler := sequential.NewScheduler("debug-stream", rsc.EventHandler) + err = events.HandleRepoStream(ctx, con, seqScheduler, nil) + if err != nil { + return err + } + + return nil + }, +} + +var compareStreamsCmd = &cli.Command{ + Name: "compare-streams", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host1", + Required: true, + }, + &cli.StringFlag{ + Name: "host2", + Required: true, + }, + }, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + h1 := cctx.String("host1") + h2 := cctx.String("host2") + + url1 := fmt.Sprintf("%s/xrpc/com.atproto.sync.subscribeRepos", h1) + url2 := fmt.Sprintf("%s/xrpc/com.atproto.sync.subscribeRepos", h2) + + d := websocket.DefaultDialer + + eventChans := []chan *comatproto.SyncSubscribeRepos_Commit{ + make(chan *comatproto.SyncSubscribeRepos_Commit, 2), + make(chan *comatproto.SyncSubscribeRepos_Commit, 2), + } + + buffers := []map[string][]*comatproto.SyncSubscribeRepos_Commit{ + make(map[string][]*comatproto.SyncSubscribeRepos_Commit), + make(map[string][]*comatproto.SyncSubscribeRepos_Commit), + } + + addToBuffer := func(n int, event *comatproto.SyncSubscribeRepos_Commit) { + buffers[n][event.Repo] = append(buffers[n][event.Repo], event) + } + + pll := func(ll *lexutil.LexLink) string { + if ll == nil { + return "" + } + return ll.String() + } + + findMatchAndRemove := func(n int, event *comatproto.SyncSubscribeRepos_Commit) (*comatproto.SyncSubscribeRepos_Commit, error) { + buf := buffers[n] + slice, ok := buf[event.Repo] + if !ok || len(slice) == 0 { + return nil, nil + } + + for i, ev := range slice { + if ev.Commit == event.Commit { + _ = pll + /* TODO: prev is no longer included in #commit messages; could use prevData or rev? + if pll(ev.Prev) != pll(event.Prev) { + // same commit different prev?? + return nil, fmt.Errorf("matched event with same commit but different prev: (%d) %d - %d", n, ev.Seq, event.Seq) + } + */ + } + + if i != 0 { + fmt.Printf("detected skipped event: %d (%d)\n", slice[0].Seq, i) + } + + slice = slice[i+1:] + buf[event.Repo] = slice + return ev, nil + } + + return nil, fmt.Errorf("did not find matching event despite having events in buffer") + } + + printCurrentDelta := func() { + var a, b int + for _, sl := range buffers[0] { + a += len(sl) + } + for _, sl := range buffers[1] { + b += len(sl) + } + + fmt.Printf("%d %d\n", a, b) + } + + printDetailedDelta := func() { + for did, sl := range buffers[0] { + osl := buffers[1][did] + if len(osl) > 0 && len(sl) > 0 { + fmt.Printf("%s had mismatched events on both streams (%d, %d)\n", did, len(sl), len(osl)) + } + + } + } + + // Create two goroutines for reading events from two URLs + for i, url := range []string{url1, url2} { + go func(i int, url string) { + con, _, err := d.Dial(url, http.Header{}) + if err != nil { + log.Error("Dial failure", "i", i, "url", url, "err", err) + os.Exit(1) + } + + ctx := context.TODO() + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + eventChans[i] <- evt + return nil + }, + // TODO: all the other Repo* event types + Error: func(evt *events.ErrorFrame) error { + return fmt.Errorf("%s: %s", evt.Error, evt.Message) + }, + } + seqScheduler := sequential.NewScheduler(fmt.Sprintf("debug-stream-%d", i+1), rsc.EventHandler) + if err := events.HandleRepoStream(ctx, con, seqScheduler, nil); err != nil { + log.Error("HandleRepoStream failure", "i", i, "url", url, "err", err) + os.Exit(1) + } + }(i, url) + } + + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT) + + // Compare events from the two URLs + for { + select { + case event := <-eventChans[0]: + partner, err := findMatchAndRemove(1, event) + if err != nil { + fmt.Println("checking for match failed: ", err) + continue + } + if partner == nil { + addToBuffer(0, event) + } else { + // the good case + fmt.Println("Match found") + } + + case event := <-eventChans[1]: + partner, err := findMatchAndRemove(0, event) + if err != nil { + fmt.Println("checking for match failed: ", err) + continue + } + if partner == nil { + addToBuffer(1, event) + } else { + // the good case + fmt.Println("Match found") + } + case <-ch: + printDetailedDelta() + /* + b, err := json.Marshal(buffers) + if err != nil { + return err + } + + fmt.Println(string(b)) + */ + return nil + } + + printCurrentDelta() + } + }, +} + +var debugFeedGenCmd = &cli.Command{ + Name: "debug-feed", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + didr := cliutil.GetDidResolver(cctx) + + uri := cctx.Args().First() + puri, err := util.ParseAtUri(uri) + if err != nil { + return err + } + + ctx := context.TODO() + + out, err := atproto.RepoGetRecord(ctx, xrpcc, "", puri.Collection, puri.Did, puri.Rkey) + if err != nil { + return fmt.Errorf("getting record: %w", err) + } + + fgr, ok := out.Value.Val.(*bsky.FeedGenerator) + if !ok { + return fmt.Errorf("invalid feedgen record") + } + + fmt.Println("Feed DID is: ", fgr.Did) + doc, err := didr.GetDocument(ctx, fgr.Did) + if err != nil { + return err + } + + fmt.Println("Got service did document:") + b, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + + var ss *did.Service + for _, s := range doc.Service { + if s.ID.String() == "#bsky_fg" { + cp := s + ss = &cp + break + } + } + + if ss == nil { + return fmt.Errorf("No '#bsky_fg' service entry found in feedgens DID document") + } + + fmt.Println("Service endpoint is: ", ss.ServiceEndpoint) + + fgclient := &xrpc.Client{ + Host: ss.ServiceEndpoint, + } + + desc, err := bsky.FeedDescribeFeedGenerator(ctx, fgclient) + if err != nil { + return err + } + + fmt.Printf("Found %d feeds at discovered endpoint\n", len(desc.Feeds)) + var found bool + for _, f := range desc.Feeds { + fmt.Println("Feed: ", f.Uri) + if f.Uri == uri { + found = true + break + } + } + + if !found { + return fmt.Errorf("specified feed was not present in linked feedGenerators 'describe' method output") + } + + skel, err := bsky.FeedGetFeedSkeleton(ctx, fgclient, "", uri, 30) + if err != nil { + return fmt.Errorf("failed to fetch feed skeleton: %w", err) + } + + if len(skel.Feed) > 30 { + return fmt.Errorf("feedgen not respecting limit param (returned %d posts)", len(skel.Feed)) + } + + if len(skel.Feed) == 0 { + return fmt.Errorf("feedgen response is empty (might be expected since we aren't authed)") + } + + fmt.Println("Feed response looks good!") + + seen := make(map[string]bool) + for _, p := range skel.Feed { + seen[p.Post] = true + } + + curs := skel.Cursor + for i := 0; i < 10 && curs != nil; i++ { + fmt.Println("Response had cursor: ", *curs) + nresp, err := bsky.FeedGetFeedSkeleton(ctx, fgclient, *curs, uri, 10) + if err != nil { + return fmt.Errorf("fetching paginated feed failed: %w", err) + } + + fmt.Printf("Got %d posts from cursored query\n", len(nresp.Feed)) + + if len(nresp.Feed) > 10 { + return fmt.Errorf("got more posts than we requested") + } + + for _, p := range nresp.Feed { + if seen[p.Post] { + return fmt.Errorf("duplicate post in response: %s", p.Post) + } + + seen[p.Post] = true + } + + if len(nresp.Feed) == 0 || nresp.Cursor == nil { + break + } + + curs = nresp.Cursor + } + + return nil + }, +} +var debugFeedViewCmd = &cli.Command{ + Name: "view-feed", + Usage: "", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, true) + if err != nil { + return err + } + + didr := cliutil.GetDidResolver(cctx) + + uri := cctx.Args().First() + puri, err := util.ParseAtUri(uri) + if err != nil { + return err + } + + ctx := context.TODO() + + out, err := atproto.RepoGetRecord(ctx, xrpcc, "", puri.Collection, puri.Did, puri.Rkey) + if err != nil { + return fmt.Errorf("getting record: %w", err) + } + + fgr, ok := out.Value.Val.(*bsky.FeedGenerator) + if !ok { + return fmt.Errorf("invalid feedgen record") + } + + doc, err := didr.GetDocument(ctx, fgr.Did) + if err != nil { + return err + } + + var ss *did.Service + for _, s := range doc.Service { + if s.ID.String() == "#bsky_fg" { + cp := s + ss = &cp + break + } + } + + if ss == nil { + return fmt.Errorf("No '#bsky_fg' service entry found in feedgens DID document") + } + + fgclient := &xrpc.Client{ + Host: ss.ServiceEndpoint, + } + + cache, err := loadCache("postcache.json") + if err != nil { + return err + } + var cacheUpdate bool + + var cursor string + getPage := func(curs string) ([]*bsky.FeedDefs_PostView, error) { + skel, err := bsky.FeedGetFeedSkeleton(ctx, fgclient, cursor, uri, 30) + if err != nil { + return nil, fmt.Errorf("failed to fetch feed skeleton: %w", err) + } + + if skel.Cursor != nil { + cursor = *skel.Cursor + } + + var posts []*bsky.FeedDefs_PostView + for _, fp := range skel.Feed { + cached, ok := cache[fp.Post] + if ok { + posts = append(posts, cached) + continue + } + fps, err := bsky.FeedGetPosts(ctx, xrpcc, []string{fp.Post}) + if err != nil { + return nil, err + } + + if len(fps.Posts) == 0 { + fmt.Println("FAILED TO GET POST: ", fp.Post) + continue + } + p := fps.Posts[0] + rec := p.Record.Val.(*bsky.FeedPost) + rec.Embed = nil // nil out embeds since they sometimes fail to json marshal... + posts = append(posts, p) + cache[fp.Post] = p + cacheUpdate = true + } + + return posts, nil + } + + printPosts := func(posts []*bsky.FeedDefs_PostView) { + for _, p := range posts { + fp, ok := p.Record.Val.(*bsky.FeedPost) + if !ok { + fmt.Printf("ERROR: Post had invalid record type: %T\n", p.Record.Val) + continue + } + text := fp.Text + text = strings.Replace(text, "\n", " ", -1) + if len(text) > 70 { + text = text[:70] + "..." + } + + dn := p.Author.Handle + if p.Author.DisplayName != nil { + dn = *p.Author.DisplayName + } + + fmt.Printf("%s: %s\n", dn, text) + } + } + + seen := make(map[string]bool) + for i := 1; i < 5; i++ { + fmt.Printf("PAGE %d - cursor: %s\n", i, cursor) + posts, err := getPage(cursor) + if err != nil { + return err + } + var alreadySeen int + for _, p := range posts { + if seen[p.Uri] { + alreadySeen++ + } + seen[p.Uri] = true + } + fmt.Printf("Already saw %d / %d posts in page 1\n", alreadySeen, len(posts)) + printPosts(posts) + fmt.Println("") + fmt.Println("") + } + + if cacheUpdate { + if err := saveCache("postcache.json", cache); err != nil { + return err + } + } + + return nil + }, +} + +func loadCache(filename string) (map[string]*bsky.FeedDefs_PostView, error) { + var data map[string]*bsky.FeedDefs_PostView + + jsonFile, err := os.Open(filename) + if err != nil { + if os.IsNotExist(err) { + return make(map[string]*bsky.FeedDefs_PostView), nil + } + + return nil, fmt.Errorf("failed to open file: %w", err) + } + defer jsonFile.Close() + + byteValue, err := io.ReadAll(jsonFile) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + + err = json.Unmarshal(byteValue, &data) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal json: %w", err) + } + + return data, nil +} + +func saveCache(filename string, data map[string]*bsky.FeedDefs_PostView) error { + file, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal json: %w", err) + } + + err = os.WriteFile(filename, file, 0644) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + + return nil +} + +var debugGetRepoCmd = &cli.Command{ + Name: "get-repo", + Flags: []cli.Flag{}, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.TODO() + + repobytes, err := comatproto.SyncGetRepo(ctx, xrpcc, cctx.Args().First(), "") + if err != nil { + return fmt.Errorf("getting repo: %w", err) + } + + rep, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(repobytes)) + if err != nil { + return err + } + + fmt.Println("Rev: ", rep.SignedCommit().Rev) + var count int + if err := rep.ForEach(ctx, "", func(k string, v cid.Cid) error { + rec, err := rep.Blockstore().Get(ctx, v) + if err != nil { + return fmt.Errorf("getting record %q: %w", k, err) + } + + count++ + _ = rec + return nil + }); err != nil { + return err + } + fmt.Printf("scanned %d records\n", count) + + return nil + }, +} + +var debugCompareReposCmd = &cli.Command{ + Name: "compare-repos", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host-1", + Usage: "method, hostname, and port of PDS instance", + Value: "https://bsky.social", + }, + &cli.StringFlag{ + Name: "host-2", + Usage: "method, hostname, and port of PDS instance", + Value: "https://bsky.network", + }, + }, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + ctx := cctx.Context + did, err := syntax.ParseAtIdentifier(cctx.Args().First()) + if err != nil { + return err + } + + wg := sync.WaitGroup{} + wg.Add(2) + + xrpc1 := xrpc.Client{ + Host: cctx.String("host-1"), + Client: &http.Client{ + Timeout: 15 * time.Minute, + }, + } + + if !cctx.IsSet("host-1") { + dir := identity.DefaultDirectory() + ident, err := dir.Lookup(ctx, did) + if err != nil { + return err + } + + xrpc1.Host = ident.PDSEndpoint() + } + + xrpc2 := xrpc.Client{ + Host: cctx.String("host-2"), + Client: &http.Client{ + Timeout: 15 * time.Minute, + }, + } + + var rep1 *repo.Repo + go func() { + defer wg.Done() + logger := log.With("host", cctx.String("host-1")) + repo1bytes, err := comatproto.SyncGetRepo(ctx, &xrpc1, did.String(), "") + if err != nil { + logger.Error("getting repo", "err", err) + os.Exit(1) + return + } + + rep1, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo1bytes)) + if err != nil { + logger.Error("reading repo", "err", err, "bytes", len(repo1bytes)) + os.Exit(1) + return + } + }() + + var rep2 *repo.Repo + go func() { + defer wg.Done() + logger := log.With("host", cctx.String("host-2")) + repo2bytes, err := comatproto.SyncGetRepo(ctx, &xrpc2, did.String(), "") + if err != nil { + logger.Error("getting repo", "err", err) + os.Exit(1) + return + } + + rep2, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo2bytes)) + if err != nil { + logger.Error("reading repo", "err", err, "bytes", len(repo2bytes)) + os.Exit(1) + return + } + }() + + wg.Wait() + + cids1 := []cid.Cid{} + blocks1 := []blocks.Block{} + + fmt.Println("Host 1 Results") + fmt.Println("Rev: ", rep1.SignedCommit().Rev) + var count int + if err := rep1.ForEach(ctx, "", func(k string, v cid.Cid) error { + cids1 = append(cids1, v) + rec, err := rep1.Blockstore().Get(ctx, v) + if err != nil { + return fmt.Errorf("getting record %q: %w", k, err) + } + blocks1 = append(blocks1, rec) + + count++ + _ = rec + return nil + }); err != nil { + return err + } + fmt.Printf("scanned %d records\n", count) + + cids2 := []cid.Cid{} + blocks2 := []blocks.Block{} + + fmt.Println("\nHost 2 Results") + fmt.Println("Rev: ", rep2.SignedCommit().Rev) + count = 0 + if err := rep2.ForEach(ctx, "", func(k string, v cid.Cid) error { + cids2 = append(cids2, v) + rec, err := rep2.Blockstore().Get(ctx, v) + if err != nil { + return fmt.Errorf("getting record %q: %w", k, err) + } + blocks2 = append(blocks2, rec) + + count++ + _ = rec + return nil + }); err != nil { + return err + } + fmt.Printf("scanned %d records\n", count) + + fmt.Println("\nComparing CIDs") + hasBadCid := false + for i, c1 := range cids1 { + if c1 != cids2[i] { + fmt.Printf("CID mismatch at index %d: %s != %s\n", i, c1, cids2[i]) + hasBadCid = true + } + } + + if !hasBadCid { + fmt.Println("All CIDs match!") + } + + fmt.Println("Comparing blocks") + hasBadBlock := false + for i, b1 := range blocks1 { + if !bytes.Equal(b1.RawData(), blocks2[i].RawData()) { + fmt.Printf("Block mismatch at index %d Host 1 Cid (%s) Host 2 Cid (%s)\n", i, b1.Cid().String(), blocks2[i].Cid().String()) + hasBadBlock = true + } + } + + if !hasBadBlock { + fmt.Println("All blocks match!") + } + + if hasBadBlock || hasBadCid { + return fmt.Errorf("mismatched blocks or cids") + } + + return nil + }, +} diff --git a/cmd/gosky/did.go b/cmd/gosky/did.go new file mode 100644 index 000000000..f60af491c --- /dev/null +++ b/cmd/gosky/did.go @@ -0,0 +1,122 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/util/cliutil" + + "github.com/urfave/cli/v2" +) + +var didCmd = &cli.Command{ + Name: "did", + Usage: "sub-commands for working with DIDs", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + didGetCmd, + didCreateCmd, + didKeyCmd, + }, +} + +var didGetCmd = &cli.Command{ + Name: "get", + ArgsUsage: ``, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "handle", + Usage: "resolve did to handle and print", + }, + }, + Action: func(cctx *cli.Context) error { + s := cliutil.GetDidResolver(cctx) + + ctx := context.TODO() + did := cctx.Args().First() + + dir := identity.DefaultDirectory() + + if cctx.Bool("handle") { + id, err := dir.LookupDID(ctx, syntax.DID(did)) + if err != nil { + return err + } + + fmt.Println(id.Handle) + return nil + } + + doc, err := s.GetDocument(context.TODO(), did) + if err != nil { + return err + } + + b, err := json.MarshalIndent(doc, "", " ") + if err != nil { + return err + } + + fmt.Println(string(b)) + return nil + }, +} + +var didCreateCmd = &cli.Command{ + Name: "create", + ArgsUsage: ` `, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "recoverydid", + }, + &cli.StringFlag{ + Name: "signingkey", + }, + }, + Action: func(cctx *cli.Context) error { + s := cliutil.GetPLCClient(cctx) + + args, err := needArgs(cctx, "handle", "service") + if err != nil { + return err + } + handle, service := args[0], args[1] + + recoverydid := cctx.String("recoverydid") + + sigkey, err := cliutil.LoadKeyFromFile(cctx.String("signingkey")) + if err != nil { + return err + } + + fmt.Println("KEYDID: ", sigkey.Public().DID()) + + ndid, err := s.CreateDID(context.TODO(), sigkey, recoverydid, handle, service) + if err != nil { + return err + } + + fmt.Println(ndid) + return nil + }, +} + +var didKeyCmd = &cli.Command{ + Name: "did-key", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "keypath", + }, + }, + Action: func(cctx *cli.Context) error { + sigkey, err := cliutil.LoadKeyFromFile(cctx.String("keypath")) + if err != nil { + return err + } + fmt.Println(sigkey.Public().DID()) + return nil + }, +} diff --git a/cmd/gosky/handle.go b/cmd/gosky/handle.go new file mode 100644 index 000000000..a12b214de --- /dev/null +++ b/cmd/gosky/handle.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/util/cliutil" + + "github.com/urfave/cli/v2" +) + +var handleCmd = &cli.Command{ + Name: "handle", + Usage: "sub-commands for working handles", + Subcommands: []*cli.Command{ + resolveHandleCmd, + updateHandleCmd, + }, +} + +var resolveHandleCmd = &cli.Command{ + Name: "resolve", + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + ctx := context.TODO() + + args, err := needArgs(cctx, "handle") + if err != nil { + return err + } + + h, err := syntax.ParseHandle(args[0]) + if err != nil { + return fmt.Errorf("resolving %q: %w", args[0], err) + } + + dir := identity.DefaultDirectory() + + res, err := dir.LookupHandle(ctx, h) + if err != nil { + return err + } + + fmt.Println(res.DID) + + return nil + }, +} + +var updateHandleCmd = &cli.Command{ + Name: "update", + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + ctx := context.TODO() + + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + args, err := needArgs(cctx, "handle") + if err != nil { + return err + } + handle := args[0] + + err = comatproto.IdentityUpdateHandle(ctx, xrpcc, &comatproto.IdentityUpdateHandle_Input{ + Handle: handle, + }) + if err != nil { + return err + } + + return nil + }, +} diff --git a/cmd/gosky/main.go b/cmd/gosky/main.go index db982bc4e..948c46123 100644 --- a/cmd/gosky/main.go +++ b/cmd/gosky/main.go @@ -4,403 +4,496 @@ import ( "bufio" "bytes" "context" - "crypto/ecdsa" "encoding/json" "fmt" + "io" + "log/slog" + "net/http" "os" + "os/signal" "strings" + "syscall" "time" - api "github.com/bluesky-social/indigo/api" - atproto "github.com/bluesky-social/indigo/api/atproto" - apibsky "github.com/bluesky-social/indigo/api/bsky" - cliutil "github.com/bluesky-social/indigo/cmd/gosky/util" - "github.com/bluesky-social/indigo/key" + _ "github.com/joho/godotenv/autoload" + + "github.com/bluesky-social/indigo/api/atproto" + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/schedulers/sequential" + "github.com/bluesky-social/indigo/handles" + lexutil "github.com/bluesky-social/indigo/lex/util" "github.com/bluesky-social/indigo/repo" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/util/cliutil" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/gorilla/websocket" + lru "github.com/hashicorp/golang-lru/v2" "github.com/ipfs/go-cid" - "github.com/lestrrat-go/jwx/jwa" - "github.com/lestrrat-go/jwx/jwk" + "github.com/ipfs/go-datastore" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipld/go-car" "github.com/polydawn/refmt/cbor" rejson "github.com/polydawn/refmt/json" "github.com/polydawn/refmt/shared" - cli "github.com/urfave/cli/v2" + "github.com/urfave/cli/v2" + "golang.org/x/time/rate" ) +var log = slog.Default().With("system", "gosky") + func main() { - app := cli.NewApp() + run(os.Args) +} + +func run(args []string) { + + app := cli.App{ + Name: "gosky", + Usage: "client CLI for atproto and bluesky", + Version: versioninfo.Short(), + } app.Flags = []cli.Flag{ &cli.StringFlag{ - Name: "pds", - Value: "", + Name: "pds-host", + Usage: "method, hostname, and port of PDS instance", + Value: "https://bsky.social", + EnvVars: []string{"ATP_PDS_HOST"}, }, &cli.StringFlag{ - Name: "auth", - Value: "bsky.auth", + Name: "auth", + Usage: "path to JSON file with ATP auth info", + Value: "bsky.auth", + EnvVars: []string{"ATP_AUTH_FILE"}, }, + &cli.StringFlag{ + Name: "plc", + Usage: "method, hostname, and port of PLC registry", + Value: "https://plc.directory", + EnvVars: []string{"ATP_PLC_HOST"}, + }, + } + + _, _, err := cliutil.SetupSlog(cliutil.LogOptions{}) + if err != nil { + fmt.Fprintf(os.Stderr, "logging setup error: %s\n", err.Error()) + os.Exit(1) + return } + app.Commands = []*cli.Command{ - actorGetSuggestionsCmd, - createSessionCmd, - deletePostCmd, + accountCmd, + adminCmd, + bskyCmd, + bgsAdminCmd, + carCmd, + debugCmd, didCmd, - feedGetCmd, - feedSetVoteCmd, - newAccountCmd, - postCmd, - refreshAuthTokenCmd, + handleCmd, syncCmd, - listAllPostsCmd, - deletePostCmd, - getNotificationsCmd, - followsCmd, - resetPasswordCmd, + createFeedGeneratorCmd, + getRecordCmd, + listAllRecordsCmd, + readRepoStreamCmd, + parseRkey, + listLabelsCmd, + verifyUserCmd, } app.RunAndExitOnError() } -var newAccountCmd = &cli.Command{ - Name: "newAccount", - Action: func(cctx *cli.Context) error { - atp, err := cliutil.GetATPClient(cctx, false) - if err != nil { - return err - } +func jsonPrint(i any) { + b, err := json.MarshalIndent(i, "", " ") + if err != nil { + panic(err) + } - email := cctx.Args().Get(0) - handle := cctx.Args().Get(1) - password := cctx.Args().Get(2) + fmt.Println(string(b)) +} - var invite *string - if inv := cctx.Args().Get(3); inv != "" { - invite = &inv +func cborToJson(data []byte) ([]byte, error) { + defer func() { + if r := recover(); r != nil { + fmt.Println("panic: ", r) + fmt.Printf("bad blob: %x\n", data) } + }() + buf := new(bytes.Buffer) + enc := rejson.NewEncoder(buf, rejson.EncodeOptions{}) - acc, err := atp.CreateAccount(context.TODO(), email, handle, password, invite) - if err != nil { - return err - } + dec := cbor.NewDecoder(cbor.DecodeOptions{}, bytes.NewReader(data)) + err := shared.TokenPump{TokenSource: dec, TokenSink: enc}.Run() + if err != nil { + return nil, err + } - b, err := json.MarshalIndent(acc, "", " ") - if err != nil { - return err - } + return buf.Bytes(), nil +} - fmt.Println(string(b)) - return nil - }, +type cachedHandle struct { + Handle string + Valid time.Time } -var createSessionCmd = &cli.Command{ - Name: "createSession", + +var readRepoStreamCmd = &cli.Command{ + Name: "read-stream", + Usage: "subscribe to a repo event stream", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "json", + }, + &cli.BoolFlag{ + Name: "unpack", + }, + &cli.BoolFlag{ + Name: "resolve-handles", + }, + &cli.Float64Flag{ + Name: "max-throughput", + Usage: "limit event consumption to a given # of req/sec (debug utility)", + }, + }, + ArgsUsage: `[ [cursor]]`, Action: func(cctx *cli.Context) error { - atp, err := cliutil.GetATPClient(cctx, false) - if err != nil { - return err - } - handle := cctx.Args().Get(0) - password := cctx.Args().Get(1) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT) + defer stop() - ses, err := atp.CreateSession(context.TODO(), handle, password) - if err != nil { - return err + arg := cctx.Args().First() + if !strings.Contains(arg, "subscribeRepos") { + arg = arg + "/xrpc/com.atproto.sync.subscribeRepos" } - - b, err := json.MarshalIndent(ses, "", " ") - if err != nil { - return err + if len(cctx.Args().Slice()) == 2 { + arg = fmt.Sprintf("%s?cursor=%s", arg, cctx.Args().Get(1)) } - fmt.Println(string(b)) - return nil - }, -} - -var postCmd = &cli.Command{ - Name: "post", - Action: func(cctx *cli.Context) error { - atp, err := cliutil.GetATPClient(cctx, true) + fmt.Fprintln(os.Stderr, "dialing: ", arg) + d := websocket.DefaultDialer + con, _, err := d.Dial(arg, http.Header{}) if err != nil { - return err + return fmt.Errorf("dial failure: %w", err) } - auth := atp.C.Auth + jsonfmt := cctx.Bool("json") + unpack := cctx.Bool("unpack") - text := strings.Join(cctx.Args().Slice(), " ") + fmt.Fprintln(os.Stderr, "Stream Started", time.Now().Format(time.RFC3339)) + defer func() { + fmt.Fprintln(os.Stderr, "Stream Exited", time.Now().Format(time.RFC3339)) + }() - resp, err := atp.RepoCreateRecord(context.TODO(), auth.Did, "app.bsky.feed.post", true, &api.PostRecord{ - Type: "app.bsky.feed.post", - Text: text, - CreatedAt: time.Now().Format("2006-01-02T15:04:05.000Z"), - }) - if err != nil { - return fmt.Errorf("failed to create post: %w", err) - } + go func() { + <-ctx.Done() + _ = con.Close() + }() - fmt.Println(resp.Cid) - fmt.Println(resp.Uri) + didr := cliutil.GetDidResolver(cctx) + hr := &handles.ProdHandleResolver{} + resolveHandles := cctx.Bool("resolve-handles") - return nil - }, -} + cache, _ := lru.New[string, *cachedHandle](10000) + resolveDid := func(ctx context.Context, did string) (string, error) { + ch, ok := cache.Get(did) + if ok { + if time.Now().Before(ch.Valid) { + return ch.Handle, nil + } + } -var didCmd = &cli.Command{ - Name: "did", - Subcommands: []*cli.Command{ - didGetCmd, - didCreateCmd, - }, -} + h, _, err := handles.ResolveDidToHandle(ctx, didr, hr, did) + if err != nil { + return "", err + } -var didGetCmd = &cli.Command{ - Name: "get", - Action: func(cctx *cli.Context) error { - s := cliutil.GetPLCClient(cctx) + cache.Add(did, &cachedHandle{ + Handle: h, + Valid: time.Now().Add(time.Minute * 10), + }) - doc, err := s.GetDocument(context.TODO(), cctx.Args().First()) - if err != nil { - return err + return h, nil } - - b, err := json.MarshalIndent(doc, "", " ") - if err != nil { - return err + var limiter *rate.Limiter + if cctx.Float64("max-throughput") > 0 { + limiter = rate.NewLimiter(rate.Limit(cctx.Float64("max-throughput")), 1) } - fmt.Println(string(b)) - return nil - }, -} - -var didCreateCmd = &cli.Command{ - Name: "create", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "recoverydid", - }, - &cli.StringFlag{ - Name: "signingkey", - }, - &cli.StringFlag{ - Name: "plc", - Value: "https://plc.staging.bsky.dev", - EnvVars: []string{"BSKY_PLC_URL"}, - }, - }, - Action: func(cctx *cli.Context) error { - s := cliutil.GetPLCClient(cctx) + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + if limiter != nil { + limiter.Wait(ctx) + } - handle := cctx.Args().Get(0) - service := cctx.Args().Get(1) + if jsonfmt { + b, err := json.Marshal(evt) + if err != nil { + return err + } + var out map[string]any + if err := json.Unmarshal(b, &out); err != nil { + return err + } + out["blocks"] = fmt.Sprintf("[%d bytes]", len(evt.Blocks)) + + if unpack { + recs, err := unpackRecords(evt.Blocks, evt.Ops) + if err != nil { + fmt.Fprintln(os.Stderr, "failed to unpack records: ", err) + } + out["records"] = recs + } + + b, err = json.Marshal(out) + if err != nil { + return err + } + fmt.Println(string(b)) + } else { + var handle string + if resolveHandles { + h, err := resolveDid(ctx, evt.Repo) + if err != nil { + fmt.Println("failed to resolve handle: ", err) + } else { + handle = h + } + } + fmt.Printf("(%d) RepoAppend: %s %s (%s)\n", evt.Seq, evt.Repo, handle, evt.Commit.String()) + + if unpack { + recs, err := unpackRecords(evt.Blocks, evt.Ops) + if err != nil { + fmt.Fprintln(os.Stderr, "failed to unpack records: ", err) + } + + for _, rec := range recs { + switch rec := rec.(type) { + case *bsky.FeedPost: + fmt.Printf("\tPost: %q\n", strings.Replace(rec.Text, "\n", " ", -1)) + } + } + + } + } - recoverydid := cctx.String("recoverydid") + return nil + }, + RepoSync: func(sync *comatproto.SyncSubscribeRepos_Sync) error { + if jsonfmt { + b, err := json.Marshal(sync) + if err != nil { + return err + } + fmt.Println(string(b)) + } else { + fmt.Printf("(%d) Sync: %s\n", sync.Seq, sync.Did) + } - sigkey, err := loadKey(cctx.String("signingkey")) - if err != nil { - return err - } + return nil - fmt.Println("KEYDID: ", sigkey.DID()) + }, + RepoInfo: func(info *comatproto.SyncSubscribeRepos_Info) error { + if jsonfmt { + b, err := json.Marshal(info) + if err != nil { + return err + } + fmt.Println(string(b)) + } else { + fmt.Printf("INFO: %s: %v\n", info.Name, info.Message) + } - ndid, err := s.CreateDID(context.TODO(), sigkey, recoverydid, handle, service) - if err != nil { - return err + return nil + }, + // TODO: all the other event types + Error: func(errf *events.ErrorFrame) error { + return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message) + }, } - - fmt.Println(ndid) - return nil + seqScheduler := sequential.NewScheduler(con.RemoteAddr().String(), rsc.EventHandler) + return events.HandleRepoStream(ctx, con, seqScheduler, log) }, } -func loadKey(kfile string) (*key.Key, error) { - kb, err := os.ReadFile(kfile) +func unpackRecords(blks []byte, ops []*atproto.SyncSubscribeRepos_RepoOp) ([]any, error) { + ctx := context.TODO() + + bstore := blockstore.NewBlockstore(datastore.NewMapDatastore()) + carr, err := car.NewCarReader(bytes.NewReader(blks)) if err != nil { return nil, err } - sk, err := jwk.ParseKey(kb) - if err != nil { - return nil, err + for { + blk, err := carr.Next() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + if err := bstore.Put(ctx, blk); err != nil { + return nil, err + } } - var spk ecdsa.PrivateKey - if err := sk.Raw(&spk); err != nil { + r, err := repo.OpenRepo(ctx, bstore, carr.Header.Roots[0]) + if err != nil { return nil, err } - curve, ok := sk.Get("crv") - if !ok { - return nil, fmt.Errorf("need a curve set") - } - return &key.Key{ - Raw: &spk, - Type: string(curve.(jwa.EllipticCurveAlgorithm)), - }, nil -} + var out []any + for _, op := range ops { + if op.Action == "create" { + _, rec, err := r.GetRecord(ctx, op.Path) + if err != nil { + return nil, err + } -var syncCmd = &cli.Command{ - Name: "sync", - Subcommands: []*cli.Command{ - syncGetRepoCmd, - syncGetRootCmd, - }, + out = append(out, rec) + } + } + + return out, nil } -var syncGetRepoCmd = &cli.Command{ - Name: "getRepo", +var getRecordCmd = &cli.Command{ + Name: "get-record", + Usage: "fetch a single record for a given repo", Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + }, &cli.BoolFlag{ Name: "raw", }, }, + ArgsUsage: ``, Action: func(cctx *cli.Context) error { - atp, err := cliutil.GetATPClient(cctx, false) - if err != nil { - return err - } - - ctx := context.TODO() - - repobytes, err := atp.SyncGetRepo(ctx, cctx.Args().First(), nil) - if err != nil { - return err - } + ctx := context.Background() + rfi := cctx.String("repo") - if cctx.Bool("raw") { - os.Stdout.Write(repobytes) - } else { - fmt.Printf("%x", repobytes) - } - - return nil - }, -} - -var syncGetRootCmd = &cli.Command{ - Name: "getRoot", - Action: func(cctx *cli.Context) error { - atp, err := cliutil.GetATPClient(cctx, false) - if err != nil { - return err - } - - ctx := context.TODO() + var repob []byte + if strings.HasPrefix(rfi, "did:") { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } - root, err := atp.SyncGetRoot(ctx, cctx.Args().First()) - if err != nil { - return err - } + rrb, err := comatproto.SyncGetRepo(ctx, xrpcc, rfi, "") + if err != nil { + return err + } + repob = rrb + } else if strings.HasPrefix(cctx.Args().First(), "at://") { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } - fmt.Println(root) + puri, err := util.ParseAtUri(cctx.Args().First()) + if err != nil { + return err + } - return nil - }, -} + out, err := comatproto.RepoGetRecord(ctx, xrpcc, "", puri.Collection, puri.Did, puri.Rkey) + if err != nil { + return err + } -func jsonPrint(i any) { - b, err := json.MarshalIndent(i, "", " ") - if err != nil { - panic(err) - } + b, err := json.MarshalIndent(out.Value.Val, "", " ") + if err != nil { + return err + } - fmt.Println(string(b)) -} + fmt.Println(string(b)) + return nil + } else if strings.HasPrefix(cctx.Args().First(), "https://bsky.app") { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } -func prettyPrintPost(p *apibsky.FeedFeedViewPost) { - fmt.Println(strings.Repeat("-", 60)) - rec := p.Post.Record.Val.(*apibsky.FeedPost) - fmt.Printf("%s (%s):\n", p.Post.Author.Handle, rec.CreatedAt) - fmt.Println(rec.Text) -} + parts := strings.Split(cctx.Args().First(), "/") + if len(parts) < 4 { + return fmt.Errorf("invalid post url") + } + rkey := parts[len(parts)-1] + did := parts[len(parts)-3] + + var collection string + switch parts[len(parts)-2] { + case "post": + collection = "app.bsky.feed.post" + case "profile": + collection = "app.bsky.actor.profile" + did = rkey + rkey = "self" + case "feed": + collection = "app.bsky.feed.generator" + default: + return fmt.Errorf("unrecognized link") + } -var feedGetCmd = &cli.Command{ - Name: "feed", - Flags: []cli.Flag{ - &cli.IntFlag{ - Name: "count", - Value: 100, - }, - &cli.StringFlag{ - Name: "author", - Usage: "specify handle of user to list their authored feed", - }, - &cli.BoolFlag{ - Name: "raw", - Usage: "print out feed in raw json", - }, - }, - Action: func(cctx *cli.Context) error { - bsky, err := cliutil.GetBskyClient(cctx, true) - if err != nil { - return err - } + atid, err := syntax.ParseAtIdentifier(did) + if err != nil { + return err + } - ctx := context.TODO() + resp, err := identity.DefaultDirectory().Lookup(ctx, atid) + if err != nil { + return err + } - raw := cctx.Bool("raw") + xrpcc.Host = resp.PDSEndpoint() - author := cctx.String("author") - if author != "" { - if author == "self" { - author = bsky.C.Auth.Did + out, err := comatproto.RepoGetRecord(ctx, xrpcc, "", collection, did, rkey) + if err != nil { + return err } - tl, err := bsky.FeedGetAuthorFeed(ctx, author, 99, nil) + b, err := json.MarshalIndent(out.Value.Val, "", " ") if err != nil { return err } - for i := len(tl.Feed) - 1; i >= 0; i-- { - it := tl.Feed[i] - if raw { - jsonPrint(it) - } else { - prettyPrintPost(it) - } - } + fmt.Println(string(b)) + return nil } else { - algo := "reverse-chronological" - tl, err := bsky.FeedGetTimeline(ctx, algo, cctx.Int("count"), nil) + fb, err := os.ReadFile(rfi) if err != nil { return err } - for i := len(tl.Feed) - 1; i >= 0; i-- { - it := tl.Feed[i] - if raw { - jsonPrint(it) - } else { - prettyPrintPost(it) - } - } + repob = fb } - return nil - - }, -} - -var actorGetSuggestionsCmd = &cli.Command{ - Name: "actorGetSuggestions", - Action: func(cctx *cli.Context) error { - bsky, err := cliutil.GetBskyClient(cctx, true) + rr, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(repob)) if err != nil { return err } - ctx := context.TODO() - - author := cctx.Args().First() - if author == "" { - author = bsky.C.Auth.Did + rc, rec, err := rr.GetRecord(ctx, cctx.Args().First()) + if err != nil { + return fmt.Errorf("get record failed: %w", err) } - resp, err := bsky.ActorGetSuggestions(ctx, 100, nil) - if err != nil { - return err + if cctx.Bool("raw") { + blk, err := rr.Blockstore().Get(ctx, rc) + if err != nil { + return err + } + + fmt.Printf("%x\n", blk.RawData()) + return nil } - b, err := json.MarshalIndent(resp.Actors, "", " ") + b, err := json.Marshal(rec) if err != nil { return err } @@ -408,113 +501,120 @@ var actorGetSuggestionsCmd = &cli.Command{ fmt.Println(string(b)) return nil - }, } -var feedSetVoteCmd = &cli.Command{ - Name: "vote", - ArgsUsage: " [direction]", - Action: func(cctx *cli.Context) error { - atpc, err := cliutil.GetATPClient(cctx, true) - if err != nil { - return err - } - - bskyc, err := cliutil.GetBskyClient(cctx, true) - if err != nil { - return err - } - - arg := cctx.Args().First() +func readDids(f string) ([]string, error) { + fi, err := os.Open(f) + if err != nil { + return nil, err + } - parts := strings.Split(arg, "/") - if len(parts) < 3 { - return fmt.Errorf("invalid post uri: %q", arg) - } - last := parts[len(parts)-1] - kind := parts[len(parts)-2] - user := parts[2] + defer fi.Close() - dir := cctx.Args().Get(1) - if dir == "" { - dir = "up" - } + scan := bufio.NewScanner(fi) + var out []string + for scan.Scan() { + out = append(out, strings.Split(scan.Text(), " ")[0]) + } - fmt.Println(user, kind, last) - ctx := context.TODO() - resp, err := api.RepoGetRecord[*api.PostRecord](atpc, ctx, user, kind, last) - if err != nil { - return fmt.Errorf("getting record: %w", err) - } + return out, nil +} - err = bskyc.FeedSetVote(ctx, &api.PostRef{Uri: resp.Uri, Cid: resp.Cid}, dir) - if err != nil { - return err +func needArgs(cctx *cli.Context, name ...string) ([]string, error) { + var out []string + for i, n := range name { + v := cctx.Args().Get(i) + if v == "" { + return nil, cli.Exit(fmt.Sprintf("argument %q required at position %d", n, i+1), 127) } - return nil - - }, + out = append(out, v) + } + return out, nil } -var refreshAuthTokenCmd = &cli.Command{ - Name: "refresh", - Usage: "refresh your auth token and overwrite it with new auth info", +var createFeedGeneratorCmd = &cli.Command{ + Name: "create-feed-gen", + Usage: "create a feed generator record", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "name", + Required: true, + }, + &cli.StringFlag{ + Name: "did", + Required: true, + }, + &cli.StringFlag{ + Name: "description", + }, + &cli.StringFlag{ + Name: "display-name", + }, + }, Action: func(cctx *cli.Context) error { - atpc, err := cliutil.GetATPClient(cctx, true) + xrpcc, err := cliutil.GetXrpcClient(cctx, true) if err != nil { return err } - a := atpc.C.Auth - a.AccessJwt = a.RefreshJwt - - ctx := context.TODO() - nauth, err := atpc.SessionRefresh(ctx) - if err != nil { - return err + rkey := cctx.String("name") + name := rkey + if dn := cctx.String("display-name"); dn != "" { + name = dn } - b, err := json.Marshal(nauth) - if err != nil { - return err - } + did := cctx.String("did") - if err := os.WriteFile(cctx.String("auth"), b, 0600); err != nil { - return err + var desc *string + if d := cctx.String("description"); d != "" { + desc = &d } - return nil - }, -} - -var deletePostCmd = &cli.Command{ - Name: "delete", - Action: func(cctx *cli.Context) error { - atpc, err := cliutil.GetATPClient(cctx, true) - if err != nil { - return err - } + ctx := context.TODO() - rkey := cctx.Args().First() + rec := &lexutil.LexiconTypeDecoder{Val: &bsky.FeedGenerator{ + CreatedAt: time.Now().Format(util.ISO8601), + Description: desc, + Did: did, + DisplayName: name, + }} + + ex, err := atproto.RepoGetRecord(ctx, xrpcc, "", "app.bsky.feed.generator", xrpcc.Auth.Did, rkey) + if err == nil { + resp, err := atproto.RepoPutRecord(ctx, xrpcc, &atproto.RepoPutRecord_Input{ + SwapRecord: ex.Cid, + Collection: "app.bsky.feed.generator", + Repo: xrpcc.Auth.Did, + Rkey: rkey, + Record: rec, + }) + if err != nil { + return err + } - if rkey == "" { - return fmt.Errorf("must specify rkey of post to delete") - } + fmt.Println(resp.Uri) + } else { + resp, err := atproto.RepoCreateRecord(ctx, xrpcc, &atproto.RepoCreateRecord_Input{ + Collection: "app.bsky.feed.generator", + Repo: xrpcc.Auth.Did, + Rkey: &rkey, + Record: rec, + }) + if err != nil { + return err + } - schema := "app.bsky.feed.post" - if strings.Contains(rkey, "/") { - parts := strings.Split(rkey, "/") - schema = parts[0] - rkey = parts[1] + fmt.Println(resp.Uri) } - return atpc.RepoDeleteRecord(context.TODO(), atpc.C.Auth.Did, schema, rkey) + return nil }, } -var listAllPostsCmd = &cli.Command{ - Name: "list", +var listAllRecordsCmd = &cli.Command{ + Name: "list", + Usage: "print all of the records for a repo or local CAR file", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "all", @@ -526,6 +626,7 @@ var listAllPostsCmd = &cli.Command{ Name: "cids", }, }, + ArgsUsage: `|`, Action: func(cctx *cli.Context) error { arg := cctx.Args().First() @@ -533,21 +634,24 @@ var listAllPostsCmd = &cli.Command{ var repob []byte if strings.HasPrefix(arg, "did:") { - atpc, err := cliutil.GetATPClient(cctx, true) + xrpcc, err := cliutil.GetXrpcClient(cctx, true) if err != nil { return err } if arg == "" { - arg = atpc.C.Auth.Did + arg = xrpcc.Auth.Did } - rrb, err := atpc.SyncGetRepo(ctx, arg, nil) + rrb, err := comatproto.SyncGetRepo(ctx, xrpcc, arg, "") if err != nil { return err } repob = rrb } else { + if len(arg) == 0 { + return cli.Exit("must specify DID string or repo path", 127) + } fb, err := os.ReadFile(arg) if err != nil { return err @@ -600,156 +704,145 @@ var listAllPostsCmd = &cli.Command{ }, } -var getNotificationsCmd = &cli.Command{ - Name: "notifs", - Flags: []cli.Flag{}, +var parseRkey = &cli.Command{ + Name: "parse-rkey", + Usage: "get the timestamp out of a record key", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "format", + Value: "rfc3339", + Usage: "output format (rfc3339 or unix)", + }, + }, + ArgsUsage: ``, Action: func(cctx *cli.Context) error { - ctx := context.TODO() - - bsky, err := cliutil.GetBskyClient(cctx, true) - if err != nil { - return err + arg := cctx.Args().First() + if arg == "" { + return cli.Exit("must specify record key", 127) } - notifs, err := apibsky.NotificationList(ctx, bsky.C, "", 50) + tid, err := syntax.ParseTID(arg) if err != nil { - return err + return cli.Exit(fmt.Errorf("failed to parse record key (%s) as a TID: %w", arg, err), 127) } - for _, n := range notifs.Notifications { - b, err := json.Marshal(n) - if err != nil { - return err - } - - fmt.Println(string(b)) + switch cctx.String("format") { + case "rfc3339": + fmt.Println(tid.Time().Format(time.RFC3339Nano)) + case "unix": + fmt.Println(tid.Time().Unix()) + default: + return cli.Exit(fmt.Errorf("unknown format: %s", cctx.String("format")), 127) } - return nil }, } -var followsCmd = &cli.Command{ - Name: "follows", - Subcommands: []*cli.Command{ - followsAddCmd, - followsListCmd, +var listLabelsCmd = &cli.Command{ + Name: "list-labels", + Usage: "list labels", + Flags: []cli.Flag{ + &cli.DurationFlag{ + Name: "since", + Value: time.Hour, + }, }, -} - -var followsAddCmd = &cli.Command{ - Name: "add", - Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { - ctx := context.TODO() - atpc, err := cliutil.GetATPClient(cctx, true) - if err != nil { - return err - } + ctx := context.TODO() - user := cctx.Args().First() + delta := cctx.Duration("since") + since := time.Now().Add(-1 * delta).UnixMilli() - follow := apibsky.GraphFollow{ - LexiconTypeID: "app.bsky.graph.follow", - CreatedAt: time.Now().Format(time.RFC3339), - Subject: &apibsky.ActorRef{ - DeclarationCid: "bafyreid27zk7lbis4zw5fz4podbvbs4fc5ivwji3dmrwa6zggnj4bnd57u", - Did: user, - }, + xrpcc := &xrpc.Client{ + Host: "https://mod.bsky.app", } - resp, err := atpc.RepoCreateRecord(ctx, atpc.C.Auth.Did, "app.bsky.graph.follow", true, &follow) - if err != nil { - return err - } + for { + out, err := atproto.TempFetchLabels(ctx, xrpcc, 100, since) + if err != nil { + return err + } - fmt.Println(resp.Uri) + for _, l := range out.Labels { + b, err := json.MarshalIndent(l, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + } + + if len(out.Labels) > 0 { + last := out.Labels[len(out.Labels)-1] + dt, err := syntax.ParseDatetime(last.Cts) + if err != nil { + return fmt.Errorf("invalid cts: %w", err) + } + since = dt.Time().UnixMilli() + } else { + break + } + } return nil }, } -var followsListCmd = &cli.Command{ - Name: "list", +var verifyUserCmd = &cli.Command{ + Name: "verify-user", + Usage: "create a feed generator record", + Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { - bskyc, err := cliutil.GetBskyClient(cctx, true) + xrpcc, err := cliutil.GetXrpcClient(cctx, true) if err != nil { return err } - user := cctx.Args().First() - if user == "" { - user = bskyc.C.Auth.Did - } - ctx := context.TODO() - resp, err := bskyc.GraphGetFollows(ctx, user, 100, nil) + arg := cctx.Args().First() + + idf, err := syntax.ParseAtIdentifier(arg) if err != nil { return err } - for _, f := range resp.Follows { - fmt.Println(f.Did, f.Handle) + ident, err := identity.DefaultDirectory().Lookup(ctx, idf) + if err != nil { + return err } - return nil - }, -} - -func cborToJson(data []byte) ([]byte, error) { - defer func() { - if r := recover(); r != nil { - fmt.Println("panic: ", r) - fmt.Printf("bad blob: %x\n", data) + profrec, err := atproto.RepoGetRecord(ctx, xrpcc, "", "app.bsky.actor.profile", ident.DID.String(), "self") + if err != nil { + return err } - }() - buf := new(bytes.Buffer) - enc := rejson.NewEncoder(buf, rejson.EncodeOptions{}) - dec := cbor.NewDecoder(cbor.DecodeOptions{}, bytes.NewReader(data)) - err := shared.TokenPump{dec, enc}.Run() - if err != nil { - return nil, err - } - - return buf.Bytes(), nil -} - -var resetPasswordCmd = &cli.Command{ - Name: "resetPassword", - Action: func(cctx *cli.Context) error { - ctx := context.TODO() + ap, ok := profrec.Value.Val.(*bsky.ActorProfile) + if !ok { + return fmt.Errorf("got wrong record type back") + } - atp, err := cliutil.GetATPClient(cctx, false) - if err != nil { - return err + var dn string + if ap.DisplayName != nil { + dn = *ap.DisplayName } - email := cctx.Args().Get(0) + rec := &lexutil.LexiconTypeDecoder{Val: &bsky.GraphVerification{ + CreatedAt: time.Now().Format(util.ISO8601), + DisplayName: dn, + Handle: ident.Handle.String(), + Subject: ident.DID.String(), + }} - err = atproto.AccountRequestPasswordReset(ctx, atp.C, &atproto.AccountRequestPasswordReset_Input{ - Email: email, + resp, err := atproto.RepoCreateRecord(ctx, xrpcc, &atproto.RepoCreateRecord_Input{ + Collection: "app.bsky.graph.verification", + Repo: xrpcc.Auth.Did, + Record: rec, }) if err != nil { return err } - inp := bufio.NewScanner(os.Stdin) - fmt.Println("Enter recovery code from email:") - inp.Scan() - code := inp.Text() - - fmt.Println("Enter new password:") - inp.Scan() - npass := inp.Text() - - if err := atproto.AccountResetPassword(ctx, atp.C, &atproto.AccountResetPassword_Input{ - Password: npass, - Token: code, - }); err != nil { - return err - } + fmt.Println(resp.Uri) return nil }, diff --git a/cmd/gosky/streamdiff.go b/cmd/gosky/streamdiff.go new file mode 100644 index 000000000..e1ab67afc --- /dev/null +++ b/cmd/gosky/streamdiff.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "fmt" + "net/http" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/events" + "github.com/bluesky-social/indigo/events/schedulers/sequential" + + "github.com/gorilla/websocket" + "github.com/urfave/cli/v2" +) + +// TODO: WIP - turns out to be more complicated than i initially thought +var streamCompareCmd = &cli.Command{ + Usage: "utility to subscribe and compare output from two repo streams", + Name: "diff-stream", + Flags: []cli.Flag{}, + ArgsUsage: ` `, + Action: func(cctx *cli.Context) error { + d := websocket.DefaultDialer + + args, err := needArgs(cctx, "hostA", "hostB") + if err != nil { + return err + } + hosta, hostb := args[0], args[1] + + cona, _, err := d.Dial(fmt.Sprintf("%s/xrpc/com.atproto.sync.subscribeRepos", hosta), http.Header{}) + if err != nil { + return fmt.Errorf("dial failure: %w", err) + } + + conb, _, err := d.Dial(fmt.Sprintf("%s/xrpc/com.atproto.sync.subscribeRepos", hostb), http.Header{}) + if err != nil { + return fmt.Errorf("dial failure: %w", err) + } + + sd := &streamDiffer{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + sd.PushA(&events.XRPCStreamEvent{ + RepoCommit: evt, + }) + return nil + }, + RepoInfo: func(evt *comatproto.SyncSubscribeRepos_Info) error { + return nil + }, + // TODO: all the other Repo* event types + Error: func(evt *events.ErrorFrame) error { + return fmt.Errorf("%s: %s", evt.Error, evt.Message) + }, + } + seqScheduler := sequential.NewScheduler("streamA", rsc.EventHandler) + err = events.HandleRepoStream(ctx, cona, seqScheduler, log) + if err != nil { + log.Error("stream A failed", "err", err) + } + }() + + go func() { + rsc := &events.RepoStreamCallbacks{ + RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + sd.PushB(&events.XRPCStreamEvent{ + RepoCommit: evt, + }) + return nil + }, + RepoInfo: func(evt *comatproto.SyncSubscribeRepos_Info) error { + return nil + }, + // TODO: all the other Repo* event types + Error: func(evt *events.ErrorFrame) error { + return fmt.Errorf("%s: %s", evt.Error, evt.Message) + }, + } + + seqScheduler := sequential.NewScheduler("streamB", rsc.EventHandler) + err = events.HandleRepoStream(ctx, conb, seqScheduler, log) + if err != nil { + log.Error("stream B failed", "err", err) + } + }() + + select {} + }, +} + +type streamDiffer struct { + Aevts []*events.XRPCStreamEvent + Bevts []*events.XRPCStreamEvent +} + +func (sd *streamDiffer) PushA(evt *events.XRPCStreamEvent) { + ix := findEvt(evt, sd.Bevts) + if ix < 0 { + sd.Aevts = append(sd.Aevts, evt) + return + } + + switch evtOp(evt) { + case "#commit": + e := evt.RepoCommit + oe := sd.Bevts[ix].RepoCommit + + if len(e.Blocks) != len(oe.Blocks) { + fmt.Printf("seq %d (A) and seq %d (B) have different carslice lengths: %d != %d", e.Seq, oe.Seq, len(e.Blocks), len(oe.Blocks)) + } + default: + } + +} + +func (sd *streamDiffer) PushB(evt *events.XRPCStreamEvent) { + +} + +func evtOp(evt *events.XRPCStreamEvent) string { + switch { + case evt.Error != nil: + return "ERROR" + case evt.RepoCommit != nil: + return "#commit" + case evt.RepoSync != nil: + return "#sync" + case evt.RepoInfo != nil: + return "#info" + default: + return "unknown" + } +} + +func sameCommit(a, b *comatproto.SyncSubscribeRepos_Commit) bool { + return a.Repo == b.Repo && a.Rev == b.Rev +} + +func findEvt(evt *events.XRPCStreamEvent, list []*events.XRPCStreamEvent) int { + evtop := evtOp(evt) + + for i, oe := range list { + if evtop != evtOp(oe) { + continue + } + + switch { + case evt.RepoCommit != nil: + if sameCommit(evt.RepoCommit, oe.RepoCommit) { + return i + } + default: + panic("unhandled event type: " + evtop) + } + } + + return -1 +} diff --git a/cmd/gosky/sync.go b/cmd/gosky/sync.go new file mode 100644 index 000000000..13c60f424 --- /dev/null +++ b/cmd/gosky/sync.go @@ -0,0 +1,159 @@ +package main + +import ( + "context" + "fmt" + "os" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/util/cliutil" + + cli "github.com/urfave/cli/v2" +) + +var syncCmd = &cli.Command{ + Name: "sync", + Usage: "sub-commands for repo sync endpoints", + Subcommands: []*cli.Command{ + syncGetRepoCmd, + syncGetRootCmd, + syncListReposCmd, + }, +} + +var syncGetRepoCmd = &cli.Command{ + Name: "get-repo", + Usage: "download repo from account's PDS to local file (or '-' for stdout). for hex combine with 'xxd -ps -u -c 0'", + ArgsUsage: ` []`, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host", + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + arg := cctx.Args().First() + if arg == "" { + return fmt.Errorf("at-identifier arg is required") + } + atid, err := syntax.ParseAtIdentifier(arg) + if err != nil { + return err + } + dir := identity.DefaultDirectory() + ident, err := dir.Lookup(ctx, atid) + if err != nil { + return err + } + + carPath := cctx.Args().Get(1) + if carPath == "" { + carPath = ident.DID.String() + ".car" + } + + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + xrpcc.Host = ident.PDSEndpoint() + if xrpcc.Host == "" { + return fmt.Errorf("no PDS endpoint for identity") + } + + if h := cctx.String("host"); h != "" { + xrpcc.Host = h + } + + log.Info("downloading", "from", xrpcc.Host, "to", carPath) + repoBytes, err := comatproto.SyncGetRepo(ctx, xrpcc, ident.DID.String(), "") + if err != nil { + return err + } + + if carPath == "-" { + _, err = os.Stdout.Write(repoBytes) + return err + } else { + return os.WriteFile(carPath, repoBytes, 0666) + } + }, +} + +var syncGetRootCmd = &cli.Command{ + Name: "get-root", + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + ctx := context.TODO() + + atid, err := syntax.ParseAtIdentifier(cctx.Args().First()) + if err != nil { + return err + } + + dir := identity.DefaultDirectory() + ident, err := dir.Lookup(ctx, atid) + if err != nil { + return err + } + + carPath := cctx.Args().Get(1) + if carPath == "" { + carPath = ident.DID.String() + ".car" + } + + xrpcc.Host = ident.PDSEndpoint() + if xrpcc.Host == "" { + return fmt.Errorf("no PDS endpoint for identity") + } + + root, err := comatproto.SyncGetHead(ctx, xrpcc, cctx.Args().First()) + if err != nil { + return err + } + + fmt.Println(root.Root) + + return nil + }, +} + +var syncListReposCmd = &cli.Command{ + Name: "list-repos", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + var curs string + for { + out, err := comatproto.SyncListRepos(context.TODO(), xrpcc, curs, 1000) + if err != nil { + return err + } + + if len(out.Repos) == 0 { + break + } + + for _, r := range out.Repos { + fmt.Println(r.Did) + } + + if out.Cursor == nil { + break + } + + curs = *out.Cursor + } + + return nil + }, +} diff --git a/cmd/gosky/util/util.go b/cmd/gosky/util/util.go deleted file mode 100644 index 8e1c95a19..000000000 --- a/cmd/gosky/util/util.go +++ /dev/null @@ -1,199 +0,0 @@ -package cliutil - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/bluesky-social/indigo/api" - "github.com/bluesky-social/indigo/xrpc" - homedir "github.com/mitchellh/go-homedir" - "github.com/urfave/cli/v2" - "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" - "gorm.io/gorm" -) - -func GetPLCClient(cctx *cli.Context) *api.PLCServer { - return &api.PLCServer{ - Host: cctx.String("plc"), - } -} - -func NewHttpClient() *http.Client { - return &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } -} - -type CliConfig struct { - filename string - PDS string -} - -func readGoskyConfig() (*CliConfig, error) { - d, err := homedir.Dir() - if err != nil { - return nil, fmt.Errorf("cannot read Home directory") - } - - f := filepath.Join(d, ".gosky") - - b, err := os.ReadFile(f) - if os.IsNotExist(err) { - return nil, nil - } - - var out CliConfig - if err := json.Unmarshal(b, &out); err != nil { - return nil, err - } - - out.filename = f - return &out, nil -} - -var Config *CliConfig - -func TryReadConfig() { - cfg, err := readGoskyConfig() - if err != nil { - fmt.Println(err) - } else { - Config = cfg - } -} - -func WriteConfig(cfg *CliConfig) error { - b, err := json.Marshal(cfg) - if err != nil { - return err - } - - return os.WriteFile(cfg.filename, b, 0664) -} - -func GetATPClient(cctx *cli.Context, authreq bool) (*api.ATProto, error) { - h := "https://bsky.social" - if pdsurl := cctx.String("pds"); pdsurl != "" { - h = pdsurl - } - - auth, err := loadAuthFromEnv(cctx, authreq) - if err != nil { - return nil, fmt.Errorf("loading auth: %w", err) - } - - return &api.ATProto{ - C: &xrpc.Client{ - Client: NewHttpClient(), - Host: h, - Auth: auth, - }, - }, nil -} - -func loadAuthFromEnv(cctx *cli.Context, req bool) (*xrpc.AuthInfo, error) { - if a := cctx.String("auth"); a != "" { - if ai, err := ReadAuth(a); err != nil && req { - return nil, err - } else { - return ai, nil - } - } - - val := os.Getenv("BSKY_AUTH") - if val == "" { - if req { - return nil, fmt.Errorf("no auth env present, BSKY_AUTH not set") - } - - return nil, nil - } - - var auth xrpc.AuthInfo - if err := json.Unmarshal([]byte(val), &auth); err != nil { - return nil, err - } - - return &auth, nil -} - -func GetBskyClient(cctx *cli.Context, authreq bool) (*api.BskyApp, error) { - h := "https://pds.staging.bsky.dev" - if pdsurl := cctx.String("pds"); pdsurl != "" { - h = pdsurl - } - - auth, err := loadAuthFromEnv(cctx, authreq) - if err != nil { - return nil, err - } - - return &api.BskyApp{ - C: &xrpc.Client{ - Host: h, - Auth: auth, - }, - }, nil -} - -func ReadAuth(fname string) (*xrpc.AuthInfo, error) { - b, err := ioutil.ReadFile(fname) - if err != nil { - return nil, err - } - var auth xrpc.AuthInfo - if err := json.Unmarshal(b, &auth); err != nil { - return nil, err - } - - return &auth, nil -} - -func SetupDatabase(dbval string) (*gorm.DB, error) { - parts := strings.SplitN(dbval, "=", 2) - if len(parts) == 1 { - return nil, fmt.Errorf("format for database string is 'DBTYPE=PARAMS'") - } - - var dial gorm.Dialector - switch parts[0] { - case "sqlite": - dial = sqlite.Open(parts[1]) - case "postgres": - dial = postgres.Open(parts[1]) - default: - return nil, fmt.Errorf("unsupported or unrecognized db type: %s", parts[0]) - } - - db, err := gorm.Open(dial, &gorm.Config{ - SkipDefaultTransaction: true, - }) - if err != nil { - return nil, err - } - - sqldb, err := db.DB() - if err != nil { - return nil, err - } - - sqldb.SetMaxIdleConns(80) - sqldb.SetMaxOpenConns(99) - sqldb.SetConnMaxIdleTime(time.Hour) - - return db, nil -} diff --git a/cmd/hepa/Dockerfile b/cmd/hepa/Dockerfile new file mode 100644 index 000000000..33f3d36a0 --- /dev/null +++ b/cmd/hepa/Dockerfile @@ -0,0 +1,37 @@ +# Run this dockerfile from the top level of the indigo git repository like: +# +# podman build -f ./cmd/hepa/Dockerfile -t hepa . + +### Compile stage +FROM golang:1.25-alpine3.22 AS build-env +RUN apk add --no-cache build-base make git + +ADD . /dockerbuild +WORKDIR /dockerbuild + +# timezone data for alpine builds +ENV GOEXPERIMENT=loopvar +RUN GIT_VERSION=$(git describe --tags --long --always) && \ + go build -tags timetzdata -o /hepa ./cmd/hepa + +### Run stage +FROM alpine:3.22 + +RUN apk add --no-cache --update dumb-init ca-certificates runit +ENTRYPOINT ["dumb-init", "--"] + +WORKDIR / +RUN mkdir -p data/hepa +COPY --from=build-env /hepa / + +# small things to make golang binaries work well under alpine +ENV GODEBUG=netdns=go +ENV TZ=Etc/UTC + +EXPOSE 2210 + +CMD ["/hepa", "run"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="atproto Auto-Moderation Service (hepa, indigo edition)" +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" diff --git a/cmd/hepa/README.md b/cmd/hepa/README.md new file mode 100644 index 000000000..43572223d --- /dev/null +++ b/cmd/hepa/README.md @@ -0,0 +1,22 @@ + +hepa (indigo edition) +===================== + +This is a simple auto-moderation daemon which wraps the automod package. This is public code. The actual version run by Bluesky is similar, but a private fork to protect methods and mechanisms. + +The name is a reference to HEPA air filters, which help keep the local atmosphere clean and healthy for humans. + +Available commands, flags, and config are documented in the usage (`--help`). + +Current features and design decisions: + +- all state (counters) and caches stored in Redis +- consumes from Relay firehose; no backfill functionality yet +- which rules are included configured at compile time +- admin access to fetch private account metadata, and to persist moderation actions, is optional. it is possible for anybody to run a `hepa` instance + +This is not a "labeling service" per say, in that it pushes labels in to an existing moderation service, and doesn't provide API endpoints or label streams. + +Performance is generally slow when first starting up, because account-level metadata is being fetched (and cached) for every firehose event. After the caches have "warmed up", events are processed faster. + +See the `automod` package's README for more documentation. diff --git a/cmd/hepa/main.go b/cmd/hepa/main.go new file mode 100644 index 000000000..9fa1ee624 --- /dev/null +++ b/cmd/hepa/main.go @@ -0,0 +1,495 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "runtime" + "strings" + "time" + + _ "github.com/joho/godotenv/autoload" + _ "net/http/pprof" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/identity/redisdir" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod/capture" + "github.com/bluesky-social/indigo/automod/consumer" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/urfave/cli/v3" + "golang.org/x/time/rate" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("exiting", "err", err) + os.Exit(-1) + } +} + +func run(args []string) error { + + app := cli.Command{ + Name: "hepa", + Usage: "automod daemon (cleans the atmosphere)", + Version: versioninfo.Short(), + } + + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "atp-relay-host", + Usage: "hostname and port of Relay to subscribe to", + Value: "wss://bsky.network", + Sources: cli.EnvVars("ATP_RELAY_HOST", "ATP_BGS_HOST"), + }, + &cli.StringFlag{ + Name: "atp-plc-host", + Usage: "method, hostname, and port of PLC registry", + Value: "https://plc.directory", + Sources: cli.EnvVars("ATP_PLC_HOST"), + }, + &cli.StringFlag{ + Name: "atp-bsky-host", + Usage: "method, hostname, and port of bsky API (appview) service. does not use auth", + Value: "https://public.api.bsky.app", + Sources: cli.EnvVars("ATP_BSKY_HOST"), + }, + &cli.StringFlag{ + Name: "atp-ozone-host", + Usage: "method, hostname, and port of ozone instance. requires ozone-admin-token as well", + Value: "https://mod.bsky.app", + Sources: cli.EnvVars("ATP_OZONE_HOST", "ATP_MOD_HOST"), + }, + &cli.StringFlag{ + Name: "ozone-did", + Usage: "DID of account to attribute ozone actions to", + Sources: cli.EnvVars("HEPA_OZONE_DID"), + }, + &cli.StringFlag{ + Name: "ozone-admin-token", + Usage: "admin authentication password for mod service", + Sources: cli.EnvVars("HEPA_OZONE_AUTH_ADMIN_TOKEN", "HEPA_MOD_AUTH_ADMIN_TOKEN"), + }, + &cli.StringFlag{ + Name: "atp-pds-host", + Usage: "method, hostname, and port of PDS (or entryway) for admin account info; uses admin auth", + Value: "https://bsky.social", + Sources: cli.EnvVars("ATP_PDS_HOST"), + }, + &cli.StringFlag{ + Name: "pds-admin-token", + Usage: "admin authentication password for PDS (or entryway)", + Sources: cli.EnvVars("HEPA_PDS_AUTH_ADMIN_TOKEN"), + }, + &cli.StringFlag{ + Name: "redis-url", + Usage: "redis connection URL", + // redis://:@localhost:6379/ + // redis://localhost:6379/0 + Sources: cli.EnvVars("HEPA_REDIS_URL"), + }, + &cli.IntFlag{ + Name: "plc-rate-limit", + Usage: "max number of requests per second to PLC registry", + Value: 100, + Sources: cli.EnvVars("HEPA_PLC_RATE_LIMIT"), + }, + &cli.StringFlag{ + Name: "sets-json-path", + Usage: "file path of JSON file containing static sets", + Sources: cli.EnvVars("HEPA_SETS_JSON_PATH"), + }, + &cli.StringFlag{ + Name: "hiveai-api-token", + Usage: "API token for Hive AI image auto-labeling", + Sources: cli.EnvVars("HIVEAI_API_TOKEN"), + }, + &cli.StringFlag{ + Name: "abyss-host", + Usage: "host for abusive image scanning API (scheme, host, port)", + Sources: cli.EnvVars("ABYSS_HOST"), + }, + &cli.StringFlag{ + Name: "abyss-password", + Usage: "admin auth password for abyss API", + Sources: cli.EnvVars("ABYSS_PASSWORD"), + }, + &cli.StringFlag{ + Name: "ruleset", + Usage: "which ruleset config to use: default, no-blobs, only-blobs", + Sources: cli.EnvVars("HEPA_RULESET"), + }, + &cli.StringFlag{ + Name: "log-level", + Usage: "log verbosity level (eg: warn, info, debug)", + Sources: cli.EnvVars("HEPA_LOG_LEVEL", "LOG_LEVEL"), + }, + &cli.StringFlag{ + Name: "ratelimit-bypass", + Usage: "HTTP header to bypass ratelimits", + Sources: cli.EnvVars("HEPA_RATELIMIT_BYPASS", "RATELIMIT_BYPASS"), + }, + &cli.IntFlag{ + Name: "firehose-parallelism", + Usage: "force a fixed number of parallel firehose workers. default (or 0) for auto-scaling; 200 works for a large instance", + Sources: cli.EnvVars("HEPA_FIREHOSE_PARALLELISM"), + }, + &cli.StringFlag{ + Name: "prescreen-host", + Usage: "hostname of prescreen server", + Sources: cli.EnvVars("HEPA_PRESCREEN_HOST"), + }, + &cli.StringFlag{ + Name: "prescreen-token", + Usage: "secret token for prescreen server", + Sources: cli.EnvVars("HEPA_PRESCREEN_TOKEN"), + }, + &cli.DurationFlag{ + Name: "report-dupe-period", + Usage: "time period within which automod will not re-report an account for the same reasonType", + Sources: cli.EnvVars("HEPA_REPORT_DUPE_PERIOD"), + Value: 1 * 24 * time.Hour, + }, + &cli.IntFlag{ + Name: "quota-mod-report-day", + Usage: "number of reports automod can file per day, for all subjects and types combined (circuit breaker)", + Sources: cli.EnvVars("HEPA_QUOTA_MOD_REPORT_DAY"), + Value: 10000, + }, + &cli.IntFlag{ + Name: "quota-mod-takedown-day", + Usage: "number of takedowns automod can action per day, for all subjects combined (circuit breaker)", + Sources: cli.EnvVars("HEPA_QUOTA_MOD_TAKEDOWN_DAY"), + Value: 200, + }, + &cli.IntFlag{ + Name: "quota-mod-action-day", + Usage: "number of misc actions automod can do per day, for all subjects combined (circuit breaker)", + Sources: cli.EnvVars("HEPA_QUOTA_MOD_ACTION_DAY"), + Value: 2000, + }, + &cli.DurationFlag{ + Name: "record-event-timeout", + Usage: "total processing time for record events (including setup, rules, and persisting)", + Sources: cli.EnvVars("HEPA_RECORD_EVENT_TIMEOUT"), + Value: 30 * time.Second, + }, + &cli.DurationFlag{ + Name: "identity-event-timeout", + Usage: "total processing time for identity and account events (including setup, rules, and persisting)", + Sources: cli.EnvVars("HEPA_IDENTITY_EVENT_TIMEOUT"), + Value: 10 * time.Second, + }, + &cli.DurationFlag{ + Name: "ozone-event-timeout", + Usage: "total processing time for ozone events (including setup, rules, and persisting)", + Sources: cli.EnvVars("HEPA_OZONE_EVENT_TIMEOUT"), + Value: 30 * time.Second, + }, + } + + app.Commands = []*cli.Command{ + runCmd, + processRecordCmd, + processRecentCmd, + captureRecentCmd, + } + + return app.Run(context.Background(), args) +} + +func configDirectory(cmd *cli.Command) (identity.Directory, error) { + baseDir := identity.BaseDirectory{ + PLCURL: cmd.String("atp-plc-host"), + HTTPClient: http.Client{ + Timeout: time.Second * 15, + }, + PLCLimiter: rate.NewLimiter(rate.Limit(cmd.Int("plc-rate-limit")), 1), + TryAuthoritativeDNS: true, + SkipDNSDomainSuffixes: []string{".bsky.social", ".staging.bsky.dev"}, + } + var dir identity.Directory + if cmd.String("redis-url") != "" { + rdir, err := redisdir.NewRedisDirectory(&baseDir, cmd.String("redis-url"), time.Hour*24, time.Minute*2, time.Minute*5, 10_000) + if err != nil { + return nil, err + } + dir = rdir + } else { + cdir := identity.NewCacheDirectory(&baseDir, 1_500_000, time.Hour*24, time.Minute*2, time.Minute*5) + dir = &cdir + } + return dir, nil +} + +func configLogger(cmd *cli.Command, writer io.Writer) *slog.Logger { + var level slog.Level + switch strings.ToLower(cmd.String("log-level")) { + case "error": + level = slog.LevelError + case "warn": + level = slog.LevelWarn + case "info": + level = slog.LevelInfo + case "debug": + level = slog.LevelDebug + default: + level = slog.LevelInfo + } + logger := slog.New(slog.NewJSONHandler(writer, &slog.HandlerOptions{ + Level: level, + })) + slog.SetDefault(logger) + return logger +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "run the hepa daemon", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "metrics-listen", + Usage: "IP or address, and port, to listen on for metrics APIs", + Value: ":3989", + Sources: cli.EnvVars("HEPA_METRICS_LISTEN"), + }, + &cli.StringFlag{ + Name: "slack-webhook-url", + // eg: https://hooks.slack.com/services/X1234 + Usage: "full URL of slack webhook", + Sources: cli.EnvVars("SLACK_WEBHOOK_URL"), + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + logger := configLogger(cmd, os.Stdout) + configOTEL("hepa") + + dir, err := configDirectory(cmd) + if err != nil { + return fmt.Errorf("failed to configure identity directory: %v", err) + } + + srv, err := NewServer( + dir, + Config{ + Logger: logger, + BskyHost: cmd.String("atp-bsky-host"), + OzoneHost: cmd.String("atp-ozone-host"), + OzoneDID: cmd.String("ozone-did"), + OzoneAdminToken: cmd.String("ozone-admin-token"), + PDSHost: cmd.String("atp-pds-host"), + PDSAdminToken: cmd.String("pds-admin-token"), + SetsFileJSON: cmd.String("sets-json-path"), + RedisURL: cmd.String("redis-url"), + SlackWebhookURL: cmd.String("slack-webhook-url"), + HiveAPIToken: cmd.String("hiveai-api-token"), + AbyssHost: cmd.String("abyss-host"), + AbyssPassword: cmd.String("abyss-password"), + RatelimitBypass: cmd.String("ratelimit-bypass"), + RulesetName: cmd.String("ruleset"), + PreScreenHost: cmd.String("prescreen-host"), + PreScreenToken: cmd.String("prescreen-token"), + ReportDupePeriod: cmd.Duration("report-dupe-period"), + QuotaModReportDay: cmd.Int("quota-mod-report-day"), + QuotaModTakedownDay: cmd.Int("quota-mod-takedown-day"), + QuotaModActionDay: cmd.Int("quota-mod-action-day"), + RecordEventTimeout: cmd.Duration("record-event-timeout"), + IdentityEventTimeout: cmd.Duration("identity-event-timeout"), + OzoneEventTimeout: cmd.Duration("ozone-event-timeout"), + }, + ) + if err != nil { + return fmt.Errorf("failed to construct server: %v", err) + } + + // ozone event consumer (if configured) + if srv.Engine.OzoneClient != nil { + oc := consumer.OzoneConsumer{ + Logger: logger.With("subsystem", "ozone-consumer"), + RedisClient: srv.RedisClient, + OzoneClient: srv.Engine.OzoneClient, + Engine: srv.Engine, + } + + go func() { + if err := oc.Run(ctx); err != nil { + slog.Error("ozone consumer failed", "err", err) + } + }() + + go func() { + if err := oc.RunPersistCursor(ctx); err != nil { + slog.Error("ozone cursor routine failed", "err", err) + } + }() + } + + // prometheus HTTP endpoint: /metrics + go func() { + runtime.SetBlockProfileRate(10) + runtime.SetMutexProfileFraction(10) + if err := srv.RunMetrics(cmd.String("metrics-listen")); err != nil { + slog.Error("failed to start metrics endpoint", "error", err) + panic(fmt.Errorf("failed to start metrics endpoint: %w", err)) + } + }() + + // firehose event consumer (note this is actually mandatory) + relayHost := cmd.String("atp-relay-host") + if relayHost != "" { + fc := consumer.FirehoseConsumer{ + Engine: srv.Engine, + Logger: logger.With("subsystem", "firehose-consumer"), + Host: cmd.String("atp-relay-host"), + Parallelism: cmd.Int("firehose-parallelism"), + RedisClient: srv.RedisClient, + } + + go func() { + if err := fc.RunPersistCursor(ctx); err != nil { + slog.Error("cursor routine failed", "err", err) + } + }() + + if err := fc.Run(ctx); err != nil { + return fmt.Errorf("failure consuming and processing firehose: %w", err) + } + } + + return nil + }, +} + +// for simple commands, not long-running daemons +func configEphemeralServer(cmd *cli.Command) (*Server, error) { + // NOTE: using stderr not stdout because some commands print to stdout + logger := configLogger(cmd, os.Stderr) + + dir, err := configDirectory(cmd) + if err != nil { + return nil, err + } + + return NewServer( + dir, + Config{ + Logger: logger, + BskyHost: cmd.String("atp-bsky-host"), + OzoneHost: cmd.String("atp-ozone-host"), + OzoneDID: cmd.String("ozone-did"), + OzoneAdminToken: cmd.String("ozone-admin-token"), + PDSHost: cmd.String("atp-pds-host"), + PDSAdminToken: cmd.String("pds-admin-token"), + SetsFileJSON: cmd.String("sets-json-path"), + RedisURL: cmd.String("redis-url"), + HiveAPIToken: cmd.String("hiveai-api-token"), + AbyssHost: cmd.String("abyss-host"), + AbyssPassword: cmd.String("abyss-password"), + RatelimitBypass: cmd.String("ratelimit-bypass"), + RulesetName: cmd.String("ruleset"), + PreScreenHost: cmd.String("prescreen-host"), + PreScreenToken: cmd.String("prescreen-token"), + }, + ) +} + +var processRecordCmd = &cli.Command{ + Name: "process-record", + Usage: "process a single record in isolation", + ArgsUsage: ``, + Flags: []cli.Flag{}, + Action: func(ctx context.Context, cmd *cli.Command) error { + uriArg := cmd.Args().First() + if uriArg == "" { + return fmt.Errorf("expected a single AT-URI argument") + } + aturi, err := syntax.ParseATURI(uriArg) + if err != nil { + return fmt.Errorf("not a valid AT-URI: %v", err) + } + + srv, err := configEphemeralServer(cmd) + if err != nil { + return err + } + + return capture.FetchAndProcessRecord(ctx, srv.Engine, aturi) + }, +} + +var processRecentCmd = &cli.Command{ + Name: "process-recent", + Usage: "fetch and process recent posts for an account", + ArgsUsage: ``, + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "limit", + Usage: "how many post records to parse", + Value: 20, + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + idArg := cmd.Args().First() + if idArg == "" { + return fmt.Errorf("expected a single AT identifier (handle or DID) argument") + } + atid, err := syntax.ParseAtIdentifier(idArg) + if err != nil { + return fmt.Errorf("not a valid handle or DID: %v", err) + } + + srv, err := configEphemeralServer(cmd) + if err != nil { + return err + } + + return capture.FetchAndProcessRecent(ctx, srv.Engine, atid, cmd.Int("limit")) + }, +} + +var captureRecentCmd = &cli.Command{ + Name: "capture-recent", + Usage: "fetch account metadata and recent posts for an account, dump JSON to stdout", + ArgsUsage: ``, + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "limit", + Usage: "how many post records to parse", + Value: 20, + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + idArg := cmd.Args().First() + if idArg == "" { + return fmt.Errorf("expected a single AT identifier (handle or DID) argument") + } + atid, err := syntax.ParseAtIdentifier(idArg) + if err != nil { + return fmt.Errorf("not a valid handle or DID: %v", err) + } + + srv, err := configEphemeralServer(cmd) + if err != nil { + return err + } + + cap, err := capture.CaptureRecent(ctx, srv.Engine, atid, cmd.Int("limit")) + if err != nil { + return err + } + + outJSON, err := json.MarshalIndent(cap, "", " ") + if err != nil { + return err + } + + fmt.Println(string(outJSON)) + return nil + }, +} diff --git a/cmd/hepa/otel.go b/cmd/hepa/otel.go new file mode 100644 index 000000000..b551325c2 --- /dev/null +++ b/cmd/hepa/otel.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "log" + "log/slog" + "os" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" +) + +var tracer = otel.Tracer("hepa") + +// Enable OTLP HTTP exporter +// For relevant environment variables: +// https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables +// At a minimum, you need to set +// OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +// TODO: this should be in cliutil or something +func configOTEL(serviceName string) { + if ep := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); ep != "" { + slog.Info("setting up trace exporter", "endpoint", ep) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + exp, err := otlptracehttp.New(ctx) + if err != nil { + log.Fatal("failed to create trace exporter", "error", err) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := exp.Shutdown(ctx); err != nil { + slog.Error("failed to shutdown trace exporter", "error", err) + } + }() + + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exp), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(serviceName), + attribute.String("env", os.Getenv("ENVIRONMENT")), // DataDog + attribute.String("environment", os.Getenv("ENVIRONMENT")), // Others + attribute.Int64("ID", 1), + )), + ) + otel.SetTracerProvider(tp) + } +} diff --git a/cmd/hepa/server.go b/cmd/hepa/server.go new file mode 100644 index 000000000..7f82b4b1a --- /dev/null +++ b/cmd/hepa/server.go @@ -0,0 +1,243 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "os" + "time" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/automod" + "github.com/bluesky-social/indigo/automod/cachestore" + "github.com/bluesky-social/indigo/automod/countstore" + "github.com/bluesky-social/indigo/automod/engine" + "github.com/bluesky-social/indigo/automod/flagstore" + "github.com/bluesky-social/indigo/automod/rules" + "github.com/bluesky-social/indigo/automod/setstore" + "github.com/bluesky-social/indigo/automod/visual" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/redis/go-redis/v9" +) + +type Server struct { + Engine *automod.Engine + RedisClient *redis.Client + + logger *slog.Logger +} + +type Config struct { + Logger *slog.Logger + BskyHost string + OzoneHost string + OzoneDID string + OzoneAdminToken string + PDSHost string + PDSAdminToken string + SetsFileJSON string + RedisURL string + SlackWebhookURL string + HiveAPIToken string + AbyssHost string + AbyssPassword string + RulesetName string + RatelimitBypass string + PreScreenHost string + PreScreenToken string + ReportDupePeriod time.Duration + QuotaModReportDay int + QuotaModTakedownDay int + QuotaModActionDay int + RecordEventTimeout time.Duration + IdentityEventTimeout time.Duration + OzoneEventTimeout time.Duration +} + +func NewServer(dir identity.Directory, config Config) (*Server, error) { + logger := config.Logger + if logger == nil { + logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })) + } + + var ozoneClient *xrpc.Client + if config.OzoneAdminToken != "" && config.OzoneDID != "" { + ozoneClient = &xrpc.Client{ + Client: util.RobustHTTPClient(), + Host: config.OzoneHost, + AdminToken: &config.OzoneAdminToken, + Auth: &xrpc.AuthInfo{}, + } + if config.RatelimitBypass != "" { + ozoneClient.Headers = make(map[string]string) + ozoneClient.Headers["x-ratelimit-bypass"] = config.RatelimitBypass + } + od, err := syntax.ParseDID(config.OzoneDID) + if err != nil { + return nil, fmt.Errorf("ozone account DID supplied was not valid: %v", err) + } + ozoneClient.Auth.Did = od.String() + logger.Info("configured ozone admin client", "did", od.String(), "ozoneHost", config.OzoneHost) + } else { + logger.Info("did not configure ozone client") + } + + var adminClient *xrpc.Client + if config.PDSAdminToken != "" { + adminClient = &xrpc.Client{ + Client: util.RobustHTTPClient(), + Host: config.PDSHost, + AdminToken: &config.PDSAdminToken, + Auth: &xrpc.AuthInfo{}, + } + if config.RatelimitBypass != "" { + adminClient.Headers = make(map[string]string) + adminClient.Headers["x-ratelimit-bypass"] = config.RatelimitBypass + } + logger.Info("configured PDS admin client", "pdsHost", config.PDSHost) + } else { + logger.Info("did not configure PDS admin client") + } + + sets := setstore.NewMemSetStore() + if config.SetsFileJSON != "" { + if err := sets.LoadFromFileJSON(config.SetsFileJSON); err != nil { + return nil, fmt.Errorf("initializing in-process setstore: %v", err) + } else { + logger.Info("loaded set config from JSON", "path", config.SetsFileJSON) + } + } + + var counters countstore.CountStore + var cache cachestore.CacheStore + var flags flagstore.FlagStore + var rdb *redis.Client + if config.RedisURL != "" { + // generic client, for cursor state + opt, err := redis.ParseURL(config.RedisURL) + if err != nil { + return nil, fmt.Errorf("parsing redis URL: %v", err) + } + rdb = redis.NewClient(opt) + // check redis connection + _, err = rdb.Ping(context.TODO()).Result() + if err != nil { + return nil, fmt.Errorf("redis ping failed: %v", err) + } + + cnt, err := countstore.NewRedisCountStore(config.RedisURL) + if err != nil { + return nil, fmt.Errorf("initializing redis countstore: %v", err) + } + counters = cnt + + csh, err := cachestore.NewRedisCacheStore(config.RedisURL, 6*time.Hour) + if err != nil { + return nil, fmt.Errorf("initializing redis cachestore: %v", err) + } + cache = csh + + flg, err := flagstore.NewRedisFlagStore(config.RedisURL) + if err != nil { + return nil, fmt.Errorf("initializing redis flagstore: %v", err) + } + flags = flg + } else { + counters = countstore.NewMemCountStore() + cache = cachestore.NewMemCacheStore(5_000, 1*time.Hour) + flags = flagstore.NewMemFlagStore() + } + + // IMPORTANT: reminder that these are the indigo-edition rules, not production rules + extraBlobRules := []automod.BlobRuleFunc{} + if config.HiveAPIToken != "" && config.RulesetName != "no-hive" { + logger.Info("configuring Hive AI image labeler") + hc := visual.NewHiveAIClient(config.HiveAPIToken) + extraBlobRules = append(extraBlobRules, hc.HiveLabelBlobRule) + + if config.PreScreenHost != "" { + psc := visual.NewPreScreenClient(config.PreScreenHost, config.PreScreenToken) + hc.PreScreenClient = psc + } + } + + if config.AbyssHost != "" && config.AbyssPassword != "" { + logger.Info("configuring abyss abusive image scanning") + ac := visual.NewAbyssClient(config.AbyssHost, config.AbyssPassword, config.RatelimitBypass) + extraBlobRules = append(extraBlobRules, ac.AbyssScanBlobRule) + } + + var ruleset automod.RuleSet + switch config.RulesetName { + case "", "default", "no-hive": + ruleset = rules.DefaultRules() + ruleset.BlobRules = append(ruleset.BlobRules, extraBlobRules...) + case "no-blobs": + ruleset = rules.DefaultRules() + ruleset.BlobRules = []automod.BlobRuleFunc{} + case "only-blobs": + ruleset.BlobRules = extraBlobRules + default: + return nil, fmt.Errorf("unknown ruleset config: %s", config.RulesetName) + } + + var notifier automod.Notifier + if config.SlackWebhookURL != "" { + notifier = &automod.SlackNotifier{ + SlackWebhookURL: config.SlackWebhookURL, + } + } + + bskyClient := xrpc.Client{ + Client: util.RobustHTTPClient(), + Host: config.BskyHost, + } + if config.RatelimitBypass != "" { + bskyClient.Headers = make(map[string]string) + bskyClient.Headers["x-ratelimit-bypass"] = config.RatelimitBypass + } + blobClient := util.RobustHTTPClient() + eng := automod.Engine{ + Logger: logger, + Directory: dir, + Counters: counters, + Sets: sets, + Flags: flags, + Cache: cache, + Rules: ruleset, + Notifier: notifier, + BskyClient: &bskyClient, + OzoneClient: ozoneClient, + AdminClient: adminClient, + BlobClient: blobClient, + Config: engine.EngineConfig{ + ReportDupePeriod: config.ReportDupePeriod, + QuotaModReportDay: config.QuotaModReportDay, + QuotaModTakedownDay: config.QuotaModTakedownDay, + QuotaModActionDay: config.QuotaModActionDay, + RecordEventTimeout: config.RecordEventTimeout, + IdentityEventTimeout: config.IdentityEventTimeout, + OzoneEventTimeout: config.OzoneEventTimeout, + }, + } + + s := &Server{ + logger: logger, + Engine: &eng, + RedisClient: rdb, + } + + return s, nil +} + +func (s *Server) RunMetrics(listen string) error { + http.Handle("/metrics", promhttp.Handler()) + return http.ListenAndServe(listen, nil) +} diff --git a/cmd/lexgen/bsky.json b/cmd/lexgen/bsky.json new file mode 100644 index 000000000..18d1f866e --- /dev/null +++ b/cmd/lexgen/bsky.json @@ -0,0 +1,26 @@ +[ + { + "package": "bsky", + "prefix": "app.bsky", + "outdir": "api/bsky", + "import": "github.com/bluesky-social/indigo/api/bsky" + }, + { + "package": "atproto", + "prefix": "com.atproto", + "outdir": "api/atproto", + "import": "github.com/bluesky-social/indigo/api/atproto" + }, + { + "package": "chat", + "prefix": "chat.bsky", + "outdir": "api/chat", + "import": "github.com/bluesky-social/indigo/api/chat" + }, + { + "package": "ozone", + "prefix": "tools.ozone", + "outdir": "api/ozone", + "import": "github.com/bluesky-social/indigo/api/ozone" + } +] diff --git a/cmd/lexgen/main.go b/cmd/lexgen/main.go index 7298bee7d..256813100 100644 --- a/cmd/lexgen/main.go +++ b/cmd/lexgen/main.go @@ -1,18 +1,18 @@ package main import ( + "errors" "fmt" "io/fs" "os" "path/filepath" "strings" - lex "github.com/bluesky-social/indigo/lex" - cli "github.com/urfave/cli/v2" + "github.com/bluesky-social/indigo/lex" + "github.com/urfave/cli/v2" ) -func findSchemas(dir string) ([]string, error) { - var out []string +func findSchemas(dir string, out []string) ([]string, error) { err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { if err != nil { return err @@ -29,13 +29,14 @@ func findSchemas(dir string) ([]string, error) { return nil }) if err != nil { - return nil, err + return out, err } return out, nil } +// for direct .json lexicon files or directories containing lexicon .json files, get one flat list of all paths to .json files func expandArgs(args []string) ([]string, error) { var out []string for _, a := range args { @@ -44,11 +45,10 @@ func expandArgs(args []string) ([]string, error) { return nil, err } if st.IsDir() { - s, err := findSchemas(a) + out, err = findSchemas(a, out) if err != nil { return nil, err } - out = append(out, s...) } else if strings.HasSuffix(a, ".json") { out = append(out, a) } @@ -64,9 +64,6 @@ func main() { &cli.StringFlag{ Name: "outdir", }, - &cli.StringFlag{ - Name: "prefix", - }, &cli.BoolFlag{ Name: "gen-server", }, @@ -76,19 +73,23 @@ func main() { &cli.StringSliceFlag{ Name: "types-import", }, + &cli.StringSliceFlag{ + Name: "external-lexicons", + }, &cli.StringFlag{ Name: "package", Value: "schemagen", }, + &cli.StringFlag{ + Name: "build", + Value: "", + }, + &cli.StringFlag{ + Name: "build-file", + Value: "", + }, } app.Action = func(cctx *cli.Context) error { - outdir := cctx.String("outdir") - if outdir == "" { - return fmt.Errorf("must specify output directory (--outdir)") - } - - prefix := cctx.String("prefix") - paths, err := expandArgs(cctx.Args().Slice()) if err != nil { return err @@ -96,6 +97,10 @@ func main() { var schemas []*lex.Schema for _, arg := range paths { + if strings.HasSuffix(arg, "com/atproto/temp/importRepo.json") { + fmt.Printf("skipping schema: %s\n", arg) + continue + } s, err := lex.ReadSchema(arg) if err != nil { return fmt.Errorf("failed to read file %q: %w", arg, err) @@ -104,14 +109,59 @@ func main() { schemas = append(schemas, s) } - pkgname := cctx.String("package") + externalPaths, err := expandArgs(cctx.StringSlice("external-lexicons")) + if err != nil { + return err + } + var externalSchemas []*lex.Schema + for _, arg := range externalPaths { + s, err := lex.ReadSchema(arg) + if err != nil { + return fmt.Errorf("failed to read file %q: %w", arg, err) + } + + externalSchemas = append(externalSchemas, s) + } - imports := map[string]string{ - "app.bsky": "github.com/bluesky-social/indigo/api/bsky", - "com.atproto": "github.com/bluesky-social/indigo/api/atproto", + buildLiteral := cctx.String("build") + buildPath := cctx.String("build-file") + var packages []lex.Package + if buildLiteral != "" { + if buildPath != "" { + return errors.New("must not set both --build and --build-file") + } + packages, err = lex.ParsePackages([]byte(buildLiteral)) + if err != nil { + return fmt.Errorf("--build error, %w", err) + } + if len(packages) == 0 { + return errors.New("--build must specify at least one Package{}") + } + } else if buildPath != "" { + blob, err := os.ReadFile(buildPath) + if err != nil { + return fmt.Errorf("--build-file error, %w", err) + } + packages, err = lex.ParsePackages(blob) + if err != nil { + return fmt.Errorf("--build-file error, %w", err) + } + if len(packages) == 0 { + return errors.New("--build-file must specify at least one Package{}") + } + } else { + return errors.New("need exactly one of --build or --build-file") } if cctx.Bool("gen-server") { + pkgname := cctx.String("package") + outdir := cctx.String("outdir") + if outdir == "" { + return fmt.Errorf("must specify output directory (--outdir)") + } + defmap := lex.BuildExtDefMap(append(schemas, externalSchemas...), packages) + _ = defmap + paths := cctx.StringSlice("types-import") importmap := make(map[string]string) for _, p := range paths { @@ -126,20 +176,7 @@ func main() { } } else { - defmap := lex.BuildExtDefMap(schemas, []string{"com.atproto", "app.bsky"}) - - lex.FixRecordReferences(schemas, defmap, prefix) - for i, s := range schemas { - if !strings.HasPrefix(s.ID, prefix) { - continue - } - - fname := filepath.Join(outdir, s.Name()+".go") - - if err := lex.GenCodeForSchema(pkgname, prefix, fname, true, s, defmap, imports); err != nil { - return fmt.Errorf("failed to process schema %q: %w", paths[i], err) - } - } + return lex.Run(schemas, externalSchemas, packages) } return nil diff --git a/cmd/netsync/dash.json b/cmd/netsync/dash.json new file mode 100644 index 000000000..7718fa5bf --- /dev/null +++ b/cmd/netsync/dash.json @@ -0,0 +1,666 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.3" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Jobs Finished" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "netsync_enqueued_jobs{}", + "legendFormat": "Jobs Enqueued", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "netsync_finished_jobs{}", + "hide": false, + "legendFormat": "Jobs Finished", + "range": true, + "refId": "B" + } + ], + "title": "Job Backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdhms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(netsync_enqueued_jobs / rate(netsync_finished_jobs{}[$__rate_interval]))", + "instant": false, + "legendFormat": "Seconds Until Completion", + "range": true, + "refId": "A" + } + ], + "title": "Time until Completion", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "netsync_bytes_processed{}", + "legendFormat": "Bytes Read", + "range": true, + "refId": "A" + } + ], + "title": "Backfilled Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(netsync_repo_clone_duration_seconds_count{status=\"success\"}[$__rate_interval])", + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(netsync_repo_clone_duration_seconds_count{status!=\"success\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{status}}", + "range": true, + "refId": "B" + } + ], + "title": "Backfilled Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{job=\"netsync\"}[$__rate_interval])", + "legendFormat": "CPU Usage per Second", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=\"netsync\"}", + "legendFormat": "Bytes Used", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "timeseries" + } + ], + "refresh": "10s", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Netsync", + "uid": "n5sI_QRSz", + "version": 13, + "weekStart": "" +} diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go new file mode 100644 index 000000000..85209f961 --- /dev/null +++ b/cmd/netsync/main.go @@ -0,0 +1,612 @@ +package main + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "log" + "log/slog" + "net/http" + "os" + "os/signal" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + _ "github.com/joho/godotenv/autoload" + + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/repo" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/ipfs/go-cid" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/urfave/cli/v3" + "golang.org/x/time/rate" +) + +func main() { + app := cli.Command{ + Name: "netsync", + Usage: "atproto network cloning tool", + Version: versioninfo.Short(), + } + + app.Flags = []cli.Flag{ + &cli.IntFlag{ + Name: "port", + Usage: "listen port for metrics server", + Value: 8753, + }, + &cli.IntFlag{ + Name: "worker-count", + Usage: "number of workers to run concurrently", + Value: 10, + }, + &cli.Float64Flag{ + Name: "checkout-limit", + Usage: "maximum number of repos per second to checkout", + Value: 4, + }, + &cli.StringFlag{ + Name: "out-dir", + Usage: "directory to write cloned repos to", + Value: "netsync-out", + }, + &cli.StringFlag{ + Name: "repo-list", + Usage: "path to file containing list of repos to clone", + Value: "repos.txt", + }, + &cli.StringFlag{ + Name: "state-file", + Usage: "path to file to write state to", + Value: "state.json", + }, + &cli.StringFlag{ + Name: "checkout-path", + Usage: "path to checkout endpoint", + Value: "https://bsky.network/xrpc/com.atproto.sync.getRepo", + }, + &cli.StringFlag{ + Name: "magic-header-key", + Usage: "header key to send with checkout request", + Value: "", + Sources: cli.EnvVars("MAGIC_HEADER_KEY"), + }, + &cli.StringFlag{ + Name: "magic-header-val", + Usage: "header value to send with checkout request", + Value: "", + Sources: cli.EnvVars("MAGIC_HEADER_VAL"), + }, + } + + app.Commands = []*cli.Command{ + { + Name: "retry", + Usage: "requeue failed repos", + Action: func(ctx context.Context, cmd *cli.Command) error { + state := &NetsyncState{ + StatePath: cmd.String("state-file"), + } + + err := state.Resume() + if err != nil { + return err + } + + // Look through finished repos for failed ones + for _, repoState := range state.FinishedRepos { + // Don't retry repos that failed due to a 400 (they've been deleted) + if strings.HasPrefix(repoState.State, "failed") && repoState.State != "failed (status: 400)" { + state.EnqueuedRepos[repoState.Repo] = &RepoState{ + Repo: repoState.Repo, + State: "enqueued", + } + } + } + + // Save state + return state.Save() + }, + }, + } + + app.Action = Netsync + + if err := app.Run(context.Background(), os.Args); err != nil { + log.Fatal(err) + } +} + +type RepoState struct { + Repo string + State string + FinishedAt time.Time +} + +type NetsyncState struct { + EnqueuedRepos map[string]*RepoState + FinishedRepos map[string]*RepoState + StatePath string + CheckoutPath string + + outDir string + magicHeaderKey string + magicHeaderVal string + + logger *slog.Logger + + lk sync.RWMutex + wg sync.WaitGroup + exit chan struct{} + limiter *rate.Limiter + workerCount int + client *http.Client +} + +type instrumentedReader struct { + source io.ReadCloser + counter prometheus.Counter +} + +func (r instrumentedReader) Read(b []byte) (int, error) { + n, err := r.source.Read(b) + r.counter.Add(float64(n)) + return n, err +} + +func (r instrumentedReader) Close() error { + var buf [32]byte + var n int + var err error + for err == nil { + n, err = r.source.Read(buf[:]) + r.counter.Add(float64(n)) + } + closeerr := r.source.Close() + if err != nil && err != io.EOF { + return err + } + return closeerr +} + +func (s *NetsyncState) Save() error { + s.lk.RLock() + defer s.lk.RUnlock() + + stateFile, err := os.OpenFile(s.StatePath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer stateFile.Close() + + stateBytes, err := json.Marshal(s) + if err != nil { + return err + } + + _, err = stateFile.Write(stateBytes) + return err +} + +func (s *NetsyncState) Resume() error { + stateFile, err := os.Open(s.StatePath) + if err != nil { + return err + } + + stateBytes, err := io.ReadAll(stateFile) + if err != nil { + return err + } + + err = json.Unmarshal(stateBytes, s) + if err != nil { + return err + } + + return nil +} + +var enqueuedJobs = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "netsync_enqueued_jobs", + Help: "Number of enqueued jobs", +}) + +func (s *NetsyncState) Dequeue() string { + s.lk.Lock() + defer s.lk.Unlock() + + enqueuedJobs.Set(float64(len(s.EnqueuedRepos))) + + for repo, state := range s.EnqueuedRepos { + if state.State == "enqueued" { + state.State = "dequeued" + return repo + } + } + + return "" +} + +var finishedJobs = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "netsync_finished_jobs", + Help: "Number of finished jobs", +}) + +func (s *NetsyncState) Finish(repo string, state string) { + s.lk.Lock() + defer s.lk.Unlock() + + s.FinishedRepos[repo] = &RepoState{ + Repo: repo, + State: state, + FinishedAt: time.Now(), + } + + finishedJobs.Set(float64(len(s.FinishedRepos))) + + delete(s.EnqueuedRepos, repo) +} + +func Netsync(ctx context.Context, cmd *cli.Command) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + logLevel := slog.LevelInfo + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: logLevel, AddSource: true})) + slog.SetDefault(slog.New(logger.Handler())) + + state := &NetsyncState{ + StatePath: cmd.String("state-file"), + CheckoutPath: cmd.String("checkout-path"), + + outDir: cmd.String("out-dir"), + workerCount: cmd.Int("worker-count"), + limiter: rate.NewLimiter(rate.Limit(cmd.Float64("checkout-limit")), 1), + magicHeaderKey: cmd.String("magic-header-key"), + magicHeaderVal: cmd.String("magic-header-val"), + + exit: make(chan struct{}), + wg: sync.WaitGroup{}, + client: &http.Client{ + Timeout: 180 * time.Second, + }, + + logger: logger, + } + + if state.magicHeaderKey != "" && state.magicHeaderVal != "" { + logger.Info("using magic header") + } + + // Create out dir + err := os.MkdirAll(state.outDir, 0755) + if err != nil { + return err + } + + // Try to resume from state file + err = state.Resume() + if state.EnqueuedRepos == nil { + state.EnqueuedRepos = make(map[string]*RepoState) + } else { + // Reset any dequeued repos + for _, repoState := range state.EnqueuedRepos { + if repoState.State == "dequeued" { + repoState.State = "enqueued" + } + } + } + + if state.FinishedRepos == nil { + state.FinishedRepos = make(map[string]*RepoState) + } + + if err != nil { + // Read repo list + repoListFile, err := os.Open(cmd.String("repo-list")) + if err != nil { + return err + } + + fileScanner := bufio.NewScanner(repoListFile) + fileScanner.Split(bufio.ScanLines) + + for fileScanner.Scan() { + repo := fileScanner.Text() + state.EnqueuedRepos[repo] = &RepoState{ + Repo: repo, + State: "enqueued", + } + } + } else { + logger.Info("Resuming from state file") + } + + // Start metrics server + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + + metricsServer := &http.Server{ + Addr: fmt.Sprintf(":%d", cmd.Int("port")), + Handler: mux, + } + + state.wg.Add(1) + go func() { + defer state.wg.Done() + if err := metricsServer.ListenAndServe(); err != http.ErrServerClosed { + logger.Error("failed to start metrics server", "err", err) + os.Exit(1) + } + logger.Info("metrics server shut down successfully") + }() + + // Start workers + for i := 0; i < state.workerCount; i++ { + state.wg.Add(1) + go func(id int) { + defer state.wg.Done() + err := state.worker(id) + if err != nil { + logger.Error("worker failed", "err", err) + } + }(i) + } + + // Check for empty queue + state.wg.Add(1) + go func() { + defer state.wg.Done() + t := time.NewTicker(30 * time.Second) + for { + select { + case <-ctx.Done(): + err := state.Save() + if err != nil { + logger.Error("failed to save state", "err", err) + } + return + case <-t.C: + err := state.Save() + if err != nil { + logger.Error("failed to save state", "err", err) + } + state.lk.RLock() + if len(state.EnqueuedRepos) == 0 { + logger.Info("no more repos to clone, shutting down") + close(state.exit) + return + } + state.lk.RUnlock() + } + } + }() + + // Trap SIGINT to trigger a shutdown. + logger.Info("listening for signals") + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + select { + case sig := <-signals: + cancel() + close(state.exit) + logger.Info("shutting down on signal", "signal", sig) + case <-ctx.Done(): + cancel() + close(state.exit) + logger.Info("shutting down on context done") + case <-state.exit: + cancel() + logger.Info("shutting down on exit signal") + } + + logger.Info("shutting down, waiting for workers to clean up...") + + if err := metricsServer.Shutdown(ctx); err != nil { + logger.Error("failed to shut down metrics server", "err", err) + } + + state.wg.Wait() + + logger.Info("shut down successfully") + + return nil + +} + +func (s *NetsyncState) worker(id int) error { + log := s.logger.With("worker", id) + log.Info("starting worker") + defer log.Info("worker stopped") + for { + select { + case <-s.exit: + log.Info("worker exiting due to exit signal") + return nil + default: + ctx := context.Background() + // Dequeue repo + repo := s.Dequeue() + if repo == "" { + // No more repos to clone + return nil + } + + // Wait for rate limiter + s.limiter.Wait(ctx) + + // Clone repo + cloneState, err := s.cloneRepo(ctx, repo) + if err != nil { + log.Error("failed to clone repo", "repo", repo, "err", err) + } + + // Update state + s.Finish(repo, cloneState) + log.Info("worker finished", "repo", repo, "status", cloneState) + } + } +} + +var repoCloneDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "netsync_repo_clone_duration_seconds", + Help: "Duration of repo clone operations", +}, []string{"status"}) + +var bytesProcessed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "netsync_bytes_processed", + Help: "Number of bytes processed", +}) + +func (s *NetsyncState) cloneRepo(ctx context.Context, did string) (cloneState string, err error) { + log := s.logger.With("repo", did, "source", "cloneRepo") + log.Info("cloning repo") + + start := time.Now() + defer func() { + duration := time.Since(start) + repoCloneDuration.WithLabelValues(cloneState).Observe(duration.Seconds()) + }() + + var url = fmt.Sprintf("%s?did=%s", s.CheckoutPath, did) + + // Clone repo + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + cloneState = "failed (request-creation)" + return cloneState, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Accept", "application/vnd.ipld.car") + req.Header.Set("User-Agent", "jaz-atproto-netsync/0.0.1") + if s.magicHeaderKey != "" && s.magicHeaderVal != "" { + req.Header.Set(s.magicHeaderKey, s.magicHeaderVal) + } + + resp, err := s.client.Do(req) + if err != nil { + cloneState = "failed (client.do)" + return cloneState, fmt.Errorf("failed to get repo: %w", err) + } + + if resp.StatusCode != http.StatusOK { + cloneState = fmt.Sprintf("failed (status: %d)", resp.StatusCode) + return cloneState, fmt.Errorf("failed to get repo: %s", resp.Status) + } + + instrumentedReader := instrumentedReader{ + source: resp.Body, + counter: bytesProcessed, + } + defer instrumentedReader.Close() + + // Write to file + outPath, err := filepath.Abs(fmt.Sprintf("%s/%s.tar.gz", s.outDir, did)) + if err != nil { + cloneState = "failed (file.abs)" + return cloneState, fmt.Errorf("failed to get absolute path: %w", err) + } + + tarFile, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + cloneState = "failed (file.open)" + return cloneState, fmt.Errorf("failed to open file: %w", err) + } + defer tarFile.Close() + + gzipWriter := gzip.NewWriter(tarFile) + defer gzipWriter.Close() + + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + numRecords := 0 + collectionsSeen := make(map[string]struct{}) + + r, err := repo.ReadRepoFromCar(ctx, instrumentedReader) + if err != nil { + log.Error("Error reading repo", "err", err) + return "failed (read-repo)", fmt.Errorf("Failed to read repo from CAR: %w", err) + } + + err = r.ForEach(ctx, "", func(path string, nodeCid cid.Cid) error { + log := log.With("path", path, "nodeCid", nodeCid) + + recordCid, rec, err := r.GetRecordBytes(ctx, path) + if err != nil { + log.Error("Error getting record", "err", err) + return nil + } + + // Verify that the record CID matches the node CID + if recordCid != nodeCid { + log.Error("Mismatch in record and node CID", "recordCID", recordCid, "nodeCID", nodeCid) + return nil + } + + parts := strings.Split(path, "/") + if len(parts) != 2 { + log.Error("Path does not have 2 parts", "path", path) + return nil + } + + collection := parts[0] + rkey := parts[1] + + numRecords++ + if _, ok := collectionsSeen[collection]; !ok { + collectionsSeen[collection] = struct{}{} + } + + asCbor, err := atdata.UnmarshalCBOR(*rec) + if err != nil { + log.Error("Error unmarshalling record", "err", err) + return fmt.Errorf("Failed to unmarshal record: %w", err) + } + + recJSON, err := json.Marshal(asCbor) + if err != nil { + log.Error("Error marshalling record to JSON", "err", err) + return fmt.Errorf("Failed to marshal record to JSON: %w", err) + } + + // Write the record directly to the tar.gz file + hdr := &tar.Header{ + Name: fmt.Sprintf("%s/%s.json", collection, rkey), + Mode: 0600, + Size: int64(len(recJSON)), + } + if err := tarWriter.WriteHeader(hdr); err != nil { + log.Error("Error writing tar header", "err", err) + return err + } + if _, err := tarWriter.Write(recJSON); err != nil { + log.Error("Error writing record to tar file", "err", err) + return err + } + + return nil + }) + if err != nil { + log.Error("Error during ForEach", "err", err) + return "failed (for-each)", fmt.Errorf("Error during ForEach: %w", err) + } + + log.Info("checkout complete", "numRecords", numRecords, "numCollections", len(collectionsSeen)) + + cloneState = "success" + return cloneState, nil +} diff --git a/cmd/palomar/Dockerfile b/cmd/palomar/Dockerfile new file mode 100644 index 000000000..2b20f1703 --- /dev/null +++ b/cmd/palomar/Dockerfile @@ -0,0 +1,37 @@ +# Run this dockerfile from the top level of the indigo git repository like: +# +# podman build -f ./cmd/palomar/Dockerfile -t palomar . + +### Compile stage +FROM golang:1.25-alpine3.22 AS build-env +RUN apk add --no-cache build-base make git + +ADD . /dockerbuild +WORKDIR /dockerbuild + +# timezone data for alpine builds +ENV GOEXPERIMENT=loopvar +RUN GIT_VERSION=$(git describe --tags --long --always) && \ + go build -tags timetzdata -o /palomar ./cmd/palomar + +### Run stage +FROM alpine:3.22 + +RUN apk add --no-cache --update dumb-init ca-certificates runit +ENTRYPOINT ["dumb-init", "--"] + +WORKDIR / +RUN mkdir -p data/palomar +COPY --from=build-env /palomar / + +# small things to make golang binaries work well under alpine +ENV GODEBUG=netdns=go +ENV TZ=Etc/UTC + +EXPOSE 3999 + +CMD ["/palomar", "run"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="atproto Search Service (for app.bsky Lexicon)" +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" diff --git a/cmd/palomar/Dockerfile.opensearch b/cmd/palomar/Dockerfile.opensearch new file mode 100644 index 000000000..079d4db29 --- /dev/null +++ b/cmd/palomar/Dockerfile.opensearch @@ -0,0 +1,3 @@ +FROM opensearchproject/opensearch:2.13.0 +RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-icu +RUN /usr/share/opensearch/bin/opensearch-plugin install --batch analysis-kuromoji diff --git a/cmd/palomar/Dockerfile.opensearch-dashboards b/cmd/palomar/Dockerfile.opensearch-dashboards new file mode 100644 index 000000000..1ab7970a1 --- /dev/null +++ b/cmd/palomar/Dockerfile.opensearch-dashboards @@ -0,0 +1,2 @@ +FROM opensearchproject/opensearch-dashboards:2.13.0 +RUN /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards diff --git a/cmd/palomar/README.md b/cmd/palomar/README.md new file mode 100644 index 000000000..d830c2c91 --- /dev/null +++ b/cmd/palomar/README.md @@ -0,0 +1,94 @@ +# Palomar + +Palomar is a backend search service for atproto, specifically the `bsky.app` post and profile record types. It works by consuming a repo event stream ("firehose") and updating an OpenSearch cluster (fork of Elasticsearch) with docs. + +Almost all the code for this service is actually in the `search/` directory at the top of this repo. + +In September 2023, this service was substantially re-written. It no longer stores records in a local database, returns only "skeleton" results (list of ATURIs or DIDs) via the HTTP API, and defines index mappings. + + +## Query String Syntax + +Currently only a simple query string syntax is supported. Double-quotes can surround phrases, `-` prefix negates a single keyword, and the following initial filters are supported: + +- `from:` will filter to results from that account, based on current (cached) identity resolution +- entire DIDs as an un-quoted keyword will result in filtering to results from that account + + +## Configuration + +Palomar uses environment variables for configuration. + +- `ATP_RELAY_HOST`: URL of firehose to subscribe to, either global Relay or individual PDS (default: `wss://bsky.network`) +- `ATP_PLC_HOST`: PLC directory for identity lookups (default: `https://plc.directory`) +- `DATABASE_URL`: connection string for database to persist firehose cursor subscription state +- `PALOMAR_BIND`: IP/port to have HTTP API listen on (default: `:3999`) +- `ES_USERNAME`: Elasticsearch username (default: `admin`) +- `ES_PASSWORD`: Password for Elasticsearch authentication +- `ES_CERT_FILE`: Optional, for TLS connections +- `ES_HOSTS`: Comma-separated list of Elasticsearch endpoints +- `ES_POST_INDEX`: name of index for post docs (default: `palomar_post`) +- `ES_PROFILE_INDEX`: name of index for profile docs (default: `palomar_profile`) +- `PALOMAR_READONLY`: Set this if the instance should act as a readonly HTTP server (no indexing) + +## HTTP API + +### Query Posts: `/xrpc/app.bsky.unspecced.searchPostsSkeleton` + +HTTP Query Params: + +- `q`: query string, required +- `limit`: integer, default 25 +- `cursor`: string, for partial pagination (uses offset, not a scroll) + +Response: + +- `posts`: array of AT-URI strings +- `hits_total`: integer; optional number of search hits (may not be populated for large result sets, eg over 10k hits) +- `cursor`: string; optionally included if there are more results that can be paginated + +### Query Profiles: `/xrpc/app.bsky.unspecced.searchActorsSkeleton` + +HTTP Query Params: + +- `q`: query string, required +- `limit`: integer, default 25 +- `cursor`: string, for partial pagination (uses offset, not a scroll) +- `typeahead`: boolean, for typeahead behavior (vs. full search) + +Response: + +- `actors`: array of AT-URI strings +- `hits_total`: integer; optional number of search hits (may not be populated for large result sets, eg over 10k hits) +- `cursor`: string; optionally included if there are more results that can be paginated + +## Development Quickstart + +Run an ephemeral opensearch instance on local port 9200, with SSL disabled, and the `analysis-icu` and `analysis-kuromoji` plugins installed, using docker: + + docker build -f Dockerfile.opensearch . -t opensearch-palomar + + # in any non-development system, obviously change this default password + docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "plugins.security.disabled=true" -e OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penSearch-Pal0mar opensearch-palomar + +See [README.opensearch.md]() for more Opensearch operational tips. + +From the top level of the repository: + + # run combined indexing and search service + make run-dev-search + + # run just the search service + READONLY=true make run-dev-search + +You'll need to get some content in to the index. An easy way to do this is to have palomar consume from the public production firehose. + +You can run test queries from the top level of the repository: + + go run ./cmd/palomar search-post "hello" + go run ./cmd/palomar search-profile "hello" + go run ./cmd/palomar search-profile -typeahead "h" + +For more commands and args: + + go run ./cmd/palomar --help diff --git a/cmd/palomar/README.opensearch.md b/cmd/palomar/README.opensearch.md new file mode 100644 index 000000000..553c7b310 --- /dev/null +++ b/cmd/palomar/README.opensearch.md @@ -0,0 +1,92 @@ + +# Basic OpenSearch Operations + +We use OpenSearch version 2.13+, with the `analysis-icu` and `analysis-kuromoji` plugins. These are included automatically on the AWS hosted version of Opensearch, otherwise you need to install: + + sudo /usr/share/opensearch/bin/opensearch-plugin install analysis-icu + sudo /usr/share/opensearch/bin/opensearch-plugin install analysis-kuromoji + sudo service opensearch restart + +If you are trying to use Elasticsearch 7.10 instead of OpenSearch, you can install the plugin with: + + sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-icu + sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-kuromoji + sudo service elasticsearch restart + +## Local Development + +With OpenSearch running locally. + +To manually drop and re-build the indices with new schemas (palomar will create these automatically if they don't exist, but this can be helpful when developing the schema itself): + + http delete :9200/palomar_post + http delete :9200/palomar_profile + http put :9200/palomar_post < post_schema.json + http put :9200/palomar_profile < profile_schema.json + +Put a single object (good for debugging): + + head -n1 examples.json | http post :9200/palomar_post/_doc/0 + http get :9200/palomar_post/_doc/0 + +Bulk insert from a file on disk: + + # esbulk is a golang CLI tool which must be installed separately + esbulk -verbose -id ident -index palomar_post -type _doc examples.json + +## Index Aliases + +To make re-indexing and schema changes easier, we can create versioned (or +time-stamped) elasticsearch indexes, and then point to them using index +aliases. The index alias updates are fast and atomic, so we can slowly build up +a new index and then cut over with no downtime. + + http put :9200/palomar_post_v04 < post_schema.json + +To do an atomic swap from one alias to a new one ("zero downtime"): + + http post :9200/_aliases << EOF + { + "actions": [ + { "remove": { "index": "palomar_post_v05", "alias": "palomar_post" }}, + { "add": { "index": "palomar_post_v06", "alias": "palomar_post" }} + ] + } + EOF + +To replace an existing ("real") index with an alias pointer, do two actions +(not truly zero-downtime, but pretty fast): + + http delete :9200/palomar_post + http put :9200/palomar_post_v03/_alias/palomar_post + +## Full-Text Querying + +A generic full-text "query string" query look like this (replace "blood" with +actual query string, and "size" field with the max results to return): + + GET /palomar_post/_search + { + "query": { + "query_string": { + "query": "blood", + "analyzer": "textIcuSearch", + "default_operator": "AND", + "analyze_wildcard": true, + "lenient": true, + "fields": ["handle^5", "text"] + } + }, + "size": 3 + } + +In the results take `.hits.hits[]._source` as the objects; `.hits.total` is the +total number of search hits. + + +## Index Debugging + +Check index size: + + http get :9200/palomar_post/_count + http get :9200/palomar_profile/_count diff --git a/cmd/palomar/docker-compose.yml b/cmd/palomar/docker-compose.yml new file mode 100644 index 000000000..ad8642c47 --- /dev/null +++ b/cmd/palomar/docker-compose.yml @@ -0,0 +1,116 @@ +version: "3.9" +services: + opensearch: + container_name: opensearch + build: + context: ../../ + dockerfile: cmd/palomar/Dockerfile.opensearch + ports: + - "9200:9200" + - "9600:9600" + environment: + - "discovery.type=single-node" + - "cluster.name=opensearch-palomar" + - "plugins.security.disabled=true" + - "bootstrap.memory_lock=true" # Disable JVM heap memory swapping + - "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # Set min and max JVM heap sizes to at least 50% of system RAM + - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penSearch-Pal0mar" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - type: bind + source: ../../data/opensearch + target: /usr/share/opensearch/data + indexer: + container_name: indexer + build: + context: ../../ + dockerfile: cmd/palomar/Dockerfile + environment: + - "GOLOG_LOG_LEVEL=info" + - "ATP_PLC_HOST=https://plc.directory" + - "ATP_BGS_HOST=wss://bsky.network" + - "ELASTIC_HOSTS=http://opensearch:9200" + - "ES_INSECURE_SSL=true" + - "ENVIRONMENT=dev" + - "ES_POST_INDEX=palomar_post_dev" + - "ES_PROFILE_INDEX=palomar_profile_dev" + - "PALOMAR_DISCOVER_REPOS=false" + - "PALOMAR_BGS_SYNC_RATE_LIMIT=20" + - "PALOMAR_INDEX_MAX_CONCURRENCY=5" + - "DATABASE_URL=sqlite:///data/palomar/search.db" + - "PALOMAR_BIND=:3997" + - "PALOMAR_METRICS_LISTEN=:3996" + depends_on: + - opensearch + ports: + - "3997:3997" + - "3996:3996" + volumes: + - type: bind + source: ../../data + target: /data + # pagerank: + # container_name: pagerank + # build: + # context: ../../ + # dockerfile: cmd/palomar/Dockerfile + # environment: + # - "GOLOG_LOG_LEVEL=info" + # - "ATP_PLC_HOST=https://plc.directory" + # - "ATP_BGS_HOST=wss://bsky.network" + # - "ELASTIC_HOSTS=http://opensearch:9200" + # - "ES_INSECURE_SSL=true" + # - "ENVIRONMENT=dev" + # - "ES_POST_INDEX=palomar_post_dev" + # - "ES_PROFILE_INDEX=palomar_profile_dev" + # - "PALOMAR_DISCOVER_REPOS=false" + # - "PALOMAR_BGS_SYNC_RATE_LIMIT=20" + # - "PALOMAR_INDEX_MAX_CONCURRENCY=5" + # - "DATABASE_URL=sqlite:///data/palomar/pagerank.db" + # - "PAGERANK_FILE=/data/palomar/pageranks.csv" + # depends_on: + # - opensearch + # volumes: + # - type: bind + # source: ../../data + # target: /data + api: + container_name: api + build: + context: ../../ + dockerfile: cmd/palomar/Dockerfile + ports: + - "3999:3999" + - "3998:3998" + environment: + - "GOLOG_LOG_LEVEL=info" + - "ATP_PLC_HOST=https://plc.directory" + - "ATP_BGS_HOST=wss://bsky.network" + - "ELASTIC_HOSTS=http://opensearch:9200" + - "ES_INSECURE_SSL=true" + - "ENVIRONMENT=dev" + - "ES_POST_INDEX=palomar_post_dev" + - "ES_PROFILE_INDEX=palomar_profile_dev" + - "DATABASE_URL=sqlite:///data/palomar/search.db" + - "PALOMAR_READONLY=true" + volumes: + - type: bind + source: ../../data + target: /data + opensearch-dashboards: + build: + context: ../../ + dockerfile: cmd/palomar/Dockerfile.opensearch-dashboards + container_name: opensearch-dashboards + ports: + - 5601:5601 + environment: + OPENSEARCH_HOSTS: '["http://opensearch:9200"]' +networks: + default: diff --git a/cmd/palomar/main.go b/cmd/palomar/main.go new file mode 100644 index 000000000..213ae2ee5 --- /dev/null +++ b/cmd/palomar/main.go @@ -0,0 +1,514 @@ +package main + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "log" + "log/slog" + "net/http" + "os" + "strings" + "time" + + _ "github.com/joho/godotenv/autoload" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/search" + "github.com/bluesky-social/indigo/util/cliutil" + + "github.com/earthboundkid/versioninfo/v2" + es "github.com/opensearch-project/opensearch-go/v2" + "github.com/urfave/cli/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" + "golang.org/x/time/rate" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("exiting", "err", err) + os.Exit(-1) + } +} + +func run(args []string) error { + + app := cli.Command{ + Name: "palomar", + Usage: "search indexing and query service (using ES or OS)", + Version: versioninfo.Short(), + } + + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "elastic-cert-file", + Usage: "certificate file path", + Sources: cli.EnvVars("ES_CERT_FILE", "ELASTIC_CERT_FILE"), + }, + &cli.BoolFlag{ + Name: "elastic-insecure-ssl", + Usage: "if true, disable SSL cert validation", + Sources: cli.EnvVars("ES_INSECURE_SSL"), + }, + &cli.StringFlag{ + Name: "elastic-username", + Usage: "elasticsearch username", + Value: "admin", + Sources: cli.EnvVars("ES_USERNAME", "ELASTIC_USERNAME"), + }, + &cli.StringFlag{ + Name: "elastic-password", + Usage: "elasticsearch password", + Value: "0penSearch-Pal0mar", + Sources: cli.EnvVars("ES_PASSWORD", "ELASTIC_PASSWORD"), + }, + &cli.StringFlag{ + Name: "elastic-hosts", + Usage: "elasticsearch hosts (schema/host/port)", + Value: "http://localhost:9200", + Sources: cli.EnvVars("ES_HOSTS", "ELASTIC_HOSTS", "OPENSEARCH_URL", "ELASTICSEARCH_URL"), + }, + &cli.StringFlag{ + Name: "es-post-index", + Usage: "ES index for 'post' documents", + Value: "palomar_post", + Sources: cli.EnvVars("ES_POST_INDEX"), + }, + &cli.StringFlag{ + Name: "es-profile-index", + Usage: "ES index for 'profile' documents", + Value: "palomar_profile", + Sources: cli.EnvVars("ES_PROFILE_INDEX"), + }, + &cli.StringFlag{ + Name: "atp-relay-host", + Usage: "hostname and port of Relay to subscribe to", + Value: "wss://bsky.network", + Sources: cli.EnvVars("ATP_RELAY_HOST", "ATP_BGS_HOST"), + }, + &cli.StringFlag{ + Name: "atp-plc-host", + Usage: "method, hostname, and port of PLC registry", + Value: "https://plc.directory", + Sources: cli.EnvVars("ATP_PLC_HOST"), + }, + &cli.IntFlag{ + Name: "max-metadb-connections", + Sources: cli.EnvVars("MAX_METADB_CONNECTIONS"), + Value: 40, + }, + &cli.StringFlag{ + Name: "log-level", + Usage: "log level (debug, info, warn, error)", + Value: "info", + Sources: cli.EnvVars("GOLOG_LOG_LEVEL", "LOG_LEVEL"), + }, + } + + app.Commands = []*cli.Command{ + runCmd, + elasticCheckCmd, + searchPostCmd, + searchProfileCmd, + } + + return app.Run(context.Background(), args) +} + +var runCmd = &cli.Command{ + Name: "run", + Usage: "combined indexing+query server", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "database-url", + Value: "sqlite://data/palomar/search.db", + Sources: cli.EnvVars("DATABASE_URL"), + }, + &cli.BoolFlag{ + Name: "readonly", + Sources: cli.EnvVars("PALOMAR_READONLY", "READONLY"), + }, + &cli.StringFlag{ + Name: "bind", + Usage: "IP or address, and port, to listen on for HTTP APIs", + Value: ":3999", + Sources: cli.EnvVars("PALOMAR_BIND"), + }, + &cli.StringFlag{ + Name: "metrics-listen", + Usage: "IP or address, and port, to listen on for metrics APIs", + Value: ":3998", + Sources: cli.EnvVars("PALOMAR_METRICS_LISTEN"), + }, + &cli.IntFlag{ + Name: "relay-sync-rate-limit", + Usage: "max repo sync (checkout) requests per second to upstream (Relay)", + Value: 8, + Sources: cli.EnvVars("PALOMAR_RELAY_SYNC_RATE_LIMIT", "PALOMAR_BGS_SYNC_RATE_LIMIT"), + }, + &cli.IntFlag{ + Name: "index-max-concurrency", + Usage: "max number of concurrent index requests (HTTP POST) to search index", + Value: 20, + Sources: cli.EnvVars("PALOMAR_INDEX_MAX_CONCURRENCY"), + }, + &cli.IntFlag{ + Name: "indexing-rate-limit", + Usage: "max number of documents per second to index", + Value: 50_000, + Sources: cli.EnvVars("PALOMAR_INDEXING_RATE_LIMIT"), + }, + &cli.IntFlag{ + Name: "plc-rate-limit", + Usage: "max number of requests per second to PLC registry", + Value: 100, + Sources: cli.EnvVars("PALOMAR_PLC_RATE_LIMIT"), + }, + &cli.BoolFlag{ + Name: "discover-repos", + Usage: "if true, discover repositories from the Relay", + Sources: cli.EnvVars("PALOMAR_DISCOVER_REPOS"), + Value: false, + }, + &cli.StringFlag{ + Name: "pagerank-file", + Sources: cli.EnvVars("PAGERANK_FILE"), + }, + &cli.StringFlag{ + Name: "bulk-posts-file", + Sources: cli.EnvVars("BULK_POSTS_FILE"), + }, + &cli.StringFlag{ + Name: "bulk-profiles-file", + Sources: cli.EnvVars("BULK_PROFILES_FILE"), + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + logLevel := slog.LevelInfo + switch cmd.String("log-level") { + case "debug": + logLevel = slog.LevelDebug + case "info": + logLevel = slog.LevelInfo + case "warn": + logLevel = slog.LevelWarn + case "error": + logLevel = slog.LevelError + } + + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: logLevel, + AddSource: true, + })) + slog.SetDefault(logger) + + readonly := cmd.Bool("readonly") + + // Enable OTLP HTTP exporter + // For relevant environment variables: + // https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables + // At a minimum, you need to set + // OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 + if ep := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); ep != "" { + slog.Info("setting up trace exporter", "endpoint", ep) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + exp, err := otlptracehttp.New(ctx) + if err != nil { + log.Fatal("failed to create trace exporter", "error", err) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := exp.Shutdown(ctx); err != nil { + slog.Error("failed to shutdown trace exporter", "error", err) + } + }() + + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exp), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("palomar"), + attribute.String("env", os.Getenv("ENVIRONMENT")), // DataDog + attribute.String("environment", os.Getenv("ENVIRONMENT")), // Others + attribute.Int64("ID", 1), + )), + ) + otel.SetTracerProvider(tp) + } + + escli, err := createEsClient(cmd) + if err != nil { + return fmt.Errorf("failed to get elasticsearch: %w", err) + } + + base := identity.BaseDirectory{ + PLCURL: cmd.String("atp-plc-host"), + HTTPClient: http.Client{ + Timeout: time.Second * 15, + }, + PLCLimiter: rate.NewLimiter(rate.Limit(cmd.Int("plc-rate-limit")), 1), + TryAuthoritativeDNS: true, + SkipDNSDomainSuffixes: []string{".bsky.social"}, + } + dir := identity.NewCacheDirectory(&base, 1_500_000, time.Hour*24, time.Minute*2, time.Minute*5) + + apiConfig := search.ServerConfig{ + Logger: logger, + ProfileIndex: cmd.String("es-profile-index"), + PostIndex: cmd.String("es-post-index"), + } + + srv, err := search.NewServer(escli, &dir, apiConfig) + if err != nil { + return err + } + + // Configure the indexer if we're not in readonly mode + if !readonly { + db, err := cliutil.SetupDatabase(cmd.String("database-url"), cmd.Int("max-metadb-connections")) + if err != nil { + return fmt.Errorf("failed to set up database: %w", err) + } + + indexerConfig := search.IndexerConfig{ + RelayHost: cmd.String("atp-relay-host"), + ProfileIndex: cmd.String("es-profile-index"), + PostIndex: cmd.String("es-post-index"), + Logger: logger, + RelaySyncRateLimit: cmd.Int("relay-sync-rate-limit"), + IndexMaxConcurrency: cmd.Int("index-max-concurrency"), + DiscoverRepos: cmd.Bool("discover-repos"), + IndexingRateLimit: cmd.Int("indexing-rate-limit"), + } + + idx, err := search.NewIndexer(db, escli, &dir, indexerConfig) + if err != nil { + return fmt.Errorf("failed to set up indexer: %w", err) + } + + srv.Indexer = idx + } + + go func() { + if err := srv.RunMetrics(cmd.String("metrics-listen")); err != nil { + slog.Error("failed to start metrics endpoint", "error", err) + panic(fmt.Errorf("failed to start metrics endpoint: %w", err)) + } + }() + + go func() { + srv.RunAPI(cmd.String("bind")) + }() + + // If we're in readonly mode, just block forever + if readonly { + select {} + } else if cmd.String("pagerank-file") != "" && srv.Indexer != nil { + // If we're not in readonly mode, and we have a pagerank file, update pageranks + ctx := context.Background() + if err := srv.Indexer.BulkIndexPageranks(ctx, cmd.String("pagerank-file")); err != nil { + return fmt.Errorf("failed to update pageranks: %w", err) + } + } else if cmd.String("bulk-posts-file") != "" && srv.Indexer != nil { + // If we're not in readonly mode, and we have a bulk posts file, index posts + ctx := context.Background() + if err := srv.Indexer.BulkIndexPosts(ctx, cmd.String("bulk-posts-file")); err != nil { + return fmt.Errorf("failed to bulk index posts: %w", err) + } + } else if cmd.String("bulk-profiles-file") != "" && srv.Indexer != nil { + // If we're not in readonly mode, and we have a bulk profiles file, index profiles + ctx := context.Background() + if err := srv.Indexer.BulkIndexProfiles(ctx, cmd.String("bulk-profiles-file")); err != nil { + return fmt.Errorf("failed to bulk index profiles: %w", err) + } + } else if srv.Indexer != nil { + // Otherwise, just run the indexer + ctx := context.Background() + if err := srv.Indexer.EnsureIndices(ctx); err != nil { + return fmt.Errorf("failed to create opensearch indices: %w", err) + } + if err := srv.Indexer.RunIndexer(ctx); err != nil { + return fmt.Errorf("failed to run indexer: %w", err) + } + } + + return nil + }, +} + +var elasticCheckCmd = &cli.Command{ + Name: "elastic-check", + Flags: []cli.Flag{}, + Action: func(ctx context.Context, cmd *cli.Command) error { + escli, err := createEsClient(cmd) + if err != nil { + return err + } + + // NOTE: this extra info check is redundant; createEsClient() already made this call and logged results + inf, err := escli.Info() + if err != nil { + return fmt.Errorf("failed to get info: %w", err) + } + defer inf.Body.Close() + if inf.IsError() { + return fmt.Errorf("failed to get info") + } + slog.Info("opensearch client connected", "client_info", inf) + + resp, err := escli.Indices.Exists([]string{cmd.String("es-profile-index"), cmd.String("es-post-index")}) + if err != nil { + return fmt.Errorf("failed to check index existence: %w", err) + } + defer resp.Body.Close() + if inf.IsError() { + return fmt.Errorf("failed to check index existence") + } + slog.Info("index existence", "resp", resp) + + return nil + + }, +} + +func printHits(resp *search.EsSearchResponse) { + fmt.Printf("%d hits in %d\n", len(resp.Hits.Hits), resp.Took) + for _, hit := range resp.Hits.Hits { + b, _ := json.Marshal(hit.Source) + fmt.Println(string(b)) + } + return +} + +var searchPostCmd = &cli.Command{ + Name: "search-post", + Usage: "run a simple query against posts index", + Action: func(ctx context.Context, cmd *cli.Command) error { + escli, err := createEsClient(cmd) + if err != nil { + return err + } + res, err := search.DoSearchPosts( + context.Background(), + identity.DefaultDirectory(), // TODO: parse PLC arg + escli, + cmd.String("es-post-index"), + &search.PostSearchParams{ + Query: strings.Join(cmd.Args().Slice(), " "), + Offset: 0, + Size: 20, + }, + ) + if err != nil { + return err + } + printHits(res) + return nil + }, +} + +var searchProfileCmd = &cli.Command{ + Name: "search-profile", + Usage: "run a simple query against posts index", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "typeahead", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + escli, err := createEsClient(cmd) + if err != nil { + return err + } + if cmd.Bool("typeahead") { + res, err := search.DoSearchProfilesTypeahead( + context.Background(), + escli, + cmd.String("es-profile-index"), + &search.ActorSearchParams{ + Query: strings.Join(cmd.Args().Slice(), " "), + Size: 10, + }, + ) + if err != nil { + return err + } + printHits(res) + } else { + res, err := search.DoSearchProfiles( + context.Background(), + identity.DefaultDirectory(), // TODO: parse PLC arg + escli, + cmd.String("es-profile-index"), + &search.ActorSearchParams{ + Query: strings.Join(cmd.Args().Slice(), " "), + Offset: 0, + Size: 20, + }, + ) + if err != nil { + return err + } + printHits(res) + } + return nil + }, +} + +func createEsClient(cmd *cli.Command) (*es.Client, error) { + + addrs := []string{} + if hosts := cmd.String("elastic-hosts"); hosts != "" { + addrs = strings.Split(hosts, ",") + } + + certfi := cmd.String("elastic-cert-file") + var cert []byte + if certfi != "" { + b, err := os.ReadFile(certfi) + if err != nil { + return nil, err + } + + cert = b + } + + insecure := cmd.Bool("elastic-insecure-ssl") + + cfg := es.Config{ + Addresses: addrs, + Username: cmd.String("elastic-username"), + Password: cmd.String("elastic-password"), + CACert: cert, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConnsPerHost: 20, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: insecure, + }, + }, + } + + escli, err := es.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to set up client: %w", err) + } + + info, err := escli.Info() + if err != nil { + return nil, fmt.Errorf("cannot get escli info: %w", err) + } + defer info.Body.Close() + slog.Debug("opensearch client initialized", "info", info) + + return escli, nil +} diff --git a/cmd/palomar/opensearch_dashboards.yml b/cmd/palomar/opensearch_dashboards.yml new file mode 100644 index 000000000..aa93bedf9 --- /dev/null +++ b/cmd/palomar/opensearch_dashboards.yml @@ -0,0 +1,4 @@ +--- +server.name: opensearch-dashboards +server.host: "0.0.0.0" +opensearch.hosts: http://opensearch:9200 diff --git a/cmd/palomar/pagerank.sh b/cmd/palomar/pagerank.sh new file mode 100644 index 000000000..255332a72 --- /dev/null +++ b/cmd/palomar/pagerank.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +export SCYLLA_KEYSPACE="${SCYLLA_KEYSPACE:-}" +export SCYLLA_HOST="${SCYLLA_HOST:-}" + +# Used by pagerank. +export FOLLOWS_FILE="/data/follows.csv" +export ACTORS_FILE="/data/actors.csv" +export OUTPUT_FILE="/data/pageranks.csv" +export EXPECTED_ACTOR_COUNT="5000000" +export RUST_LOG="info" + +# Used by palomar. +export PAGERANK_FILE="${OUTPUT_FILE}" +export PALOMAR_INDEXING_RATE_LIMIT="10000" + +function run_pagerank { + # Check that the required environment variables are set. + if [[ "${SCYLLA_KEYSPACE}" == "" ]]; then + echo "SCYLLA_KEYSPACE is not set" + exit 1 + fi + + if [[ "${SCYLLA_HOST}" == "" ]]; then + echo "SCYLLA_HOST is not set" + exit 1 + fi + + # Dump the tables to CSV files. + rm --force "${FOLLOWS_FILE}" + cqlsh \ + "--keyspace=${SCYLLA_KEYSPACE}" \ + "--request-timeout=1200" \ + ---execute "COPY follows (actor_did, subject_did) TO '${FOLLOWS_FILE}' WITH HEADER = FALSE;" \ + "${SCYLLA_HOST}" + + rm --force "${ACTORS_FILE}" + cqlsh \ + "--keyspace=${SCYLLA_KEYSPACE}" \ + "--request-timeout=1200" \ + ---execute "COPY actors (did) TO '${ACTORS_FILE}' WITH HEADER = FALSE;" \ + "${SCYLLA_HOST}" + + # Run the pagerank file which reads in the table CSV files and outputs a CSV. + /usr/local/bin/pagerank + + # Run palomar with the pagerank CSV file. + /palomar run +} + +while true; do + run_pagerank + sleep 24h +done diff --git a/cmd/pds/main.go b/cmd/pds/main.go deleted file mode 100644 index 9be56ff87..000000000 --- a/cmd/pds/main.go +++ /dev/null @@ -1,179 +0,0 @@ -package main - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "encoding/json" - "fmt" - "os" - - "github.com/bluesky-social/indigo/api" - "github.com/bluesky-social/indigo/carstore" - "github.com/bluesky-social/indigo/plc" - server "github.com/bluesky-social/indigo/server" - "github.com/lestrrat-go/jwx/v2/jwk" - "github.com/urfave/cli/v2" - "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" - "gorm.io/gorm" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/jaeger" - "go.opentelemetry.io/otel/sdk/resource" - tracesdk "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.4.0" - "gorm.io/plugin/opentelemetry/tracing" -) - -func main() { - app := cli.NewApp() - - app.Flags = []cli.Flag{ - &cli.BoolFlag{ - Name: "jaeger", - }, - &cli.BoolFlag{ - // Temp flag for testing, eventually will just pass db connection strings here - Name: "postgres", - }, - &cli.BoolFlag{ - Name: "dbtracing", - }, - &cli.StringFlag{ - Name: "pdshost", - Usage: "hostname of the pds", - Value: "localhost:4989", - }, - &cli.StringFlag{ - Name: "plc", - Usage: "hostname of the plc", - }, - } - - app.Commands = []*cli.Command{ - generateKeyCmd, - } - - app.Action = func(cctx *cli.Context) error { - - if cctx.Bool("jaeger") { - url := "http://localhost:14268/api/traces" - exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(url))) - if err != nil { - return err - } - tp := tracesdk.NewTracerProvider( - // Always be sure to batch in production. - tracesdk.WithBatcher(exp), - // Record information about this application in a Resource. - tracesdk.WithResource(resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String("pds"), - attribute.String("environment", "test"), - attribute.Int64("ID", 1), - )), - ) - - otel.SetTracerProvider(tp) - } - - pgdb := cctx.Bool("postgres") - dbtracing := cctx.Bool("dbtracing") - - var pdsdial gorm.Dialector - if pgdb { - dsn := "host=localhost user=postgres password=password dbname=pdsdb port=5432 sslmode=disable" - pdsdial = postgres.Open(dsn) - } else { - pdsdial = sqlite.Open("pdsdata/pds.db") - } - db, err := gorm.Open(pdsdial, &gorm.Config{}) - if err != nil { - return err - } - - if dbtracing { - if err := db.Use(tracing.NewPlugin()); err != nil { - return err - } - } - - var cardial gorm.Dialector - if pgdb { - dsn2 := "host=localhost user=postgres password=password dbname=cardb port=5432 sslmode=disable" - cardial = postgres.Open(dsn2) - } else { - cardial = sqlite.Open("pdsdata/carstore.db") - } - carstdb, err := gorm.Open(cardial, &gorm.Config{}) - if err != nil { - return err - } - - if dbtracing { - if err := carstdb.Use(tracing.NewPlugin()); err != nil { - return err - } - } - - cs, err := carstore.NewCarStore(carstdb, "pdsdata/carstore") - if err != nil { - return err - } - - var didr plc.PLCClient - if plchost := cctx.String("plc"); plchost != "" { - didr = &api.PLCServer{Host: plchost} - } else { - didr = plc.NewFakeDid(db) - } - - pdshost := cctx.String("pdshost") - srv, err := server.NewServer(db, cs, "server.key", ".pdstest", pdshost, didr, []byte("jwtsecretplaceholder")) - if err != nil { - return err - } - - return srv.RunAPI(":4989") - } - - app.RunAndExitOnError() -} - -var generateKeyCmd = &cli.Command{ - Name: "gen-key", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "filename", - Value: "server.key", - }, - }, - Action: func(cctx *cli.Context) error { - raw, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return fmt.Errorf("failed to generate new ECDSA private key: %s", err) - } - - key, err := jwk.FromRaw(raw) - if err != nil { - return fmt.Errorf("failed to create ECDSA key: %s", err) - } - - if _, ok := key.(jwk.ECDSAPrivateKey); !ok { - return fmt.Errorf("expected jwk.ECDSAPrivateKey, got %T", key) - } - - key.Set(jwk.KeyIDKey, "mykey") - - buf, err := json.MarshalIndent(key, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal key into JSON: %w", err) - } - - fname := cctx.String("filename") - - return os.WriteFile(fname, buf, 0664) - }, -} diff --git a/cmd/querycheck/main.go b/cmd/querycheck/main.go new file mode 100644 index 000000000..27779001d --- /dev/null +++ b/cmd/querycheck/main.go @@ -0,0 +1,174 @@ +package main + +import ( + "context" + "fmt" + "log" + "log/slog" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + + _ "net/http/pprof" + + "github.com/bluesky-social/indigo/querycheck" + "github.com/bluesky-social/indigo/util/tracing" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/labstack/echo-contrib/pprof" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/urfave/cli/v3" + "go.opentelemetry.io/otel/trace" +) + +func main() { + app := cli.Command{ + Name: "querycheck", + Usage: "a postgresql query plan checker", + Version: versioninfo.Short(), + } + + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "postgres-url", + Usage: "postgres url for storing events", + Value: "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable", + Sources: cli.EnvVars("POSTGRES_URL"), + }, + &cli.IntFlag{ + Name: "port", + Usage: "port to serve metrics on", + Value: 8080, + Sources: cli.EnvVars("PORT"), + }, + &cli.StringFlag{ + Name: "auth-token", + Usage: "auth token for accessing the querycheck api", + Value: "", + Sources: cli.EnvVars("AUTH_TOKEN"), + }, + } + + app.Action = Querycheck + + if err := app.Run(context.Background(), os.Args); err != nil { + log.Fatal(err) + } +} + +var tracer trace.Tracer + +// Querycheck is the main function for querycheck +func Querycheck(ctx context.Context, cmd *cli.Command) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Trap SIGINT to trigger a shutdown. + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelInfo, + })) + defer func() { + logger.Info("main function teardown") + }() + + logger = logger.With("source", "querycheck_main") + logger.Info("starting querycheck") + + // Registers a tracer Provider globally if the exporter endpoint is set + if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" { + logger.Info("initializing tracer") + shutdown, err := tracing.InstallExportPipeline(ctx, "Querycheck", 1) + if err != nil { + log.Fatal(err) + } + defer func() { + if err := shutdown(ctx); err != nil { + log.Fatal(err) + } + }() + } + + wg := sync.WaitGroup{} + + // HTTP Server setup and Middleware Plumbing + e := echo.New() + e.HideBanner = true + e.HidePort = true + pprof.Register(e) + e.GET("/metrics", echo.WrapHandler(promhttp.Handler())) + e.Use(middleware.LoggerWithConfig(middleware.DefaultLoggerConfig)) + + // Start the query checker + querychecker, err := querycheck.NewQuerychecker(ctx, cmd.String("postgres-url")) + if err != nil { + log.Fatalf("failed to create querychecker: %+v\n", err) + } + + // getLikeCountQuery := `SELECT * + // FROM like_counts + // WHERE actor_did = 'did:plc:q6gjnaw2blty4crticxkmujt' + // AND ns = 'app.bsky.feed.post' + // AND rkey = '3k3jf5lgbsw24' + // LIMIT 1;` + + // querychecker.AddQuery(ctx, "get_like_count", getLikeCountQuery, time.Second*20) + + err = querychecker.Start() + if err != nil { + log.Fatalf("failed to start querychecker: %+v\n", err) + } + + e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + if cmd.String("auth-token") != "" && c.Request().Header.Get("Authorization") != cmd.String("auth-token") { + return c.String(http.StatusUnauthorized, "unauthorized") + } + return next(c) + } + }) + + e.GET("/query", querychecker.HandleGetQuery) + e.GET("/queries", querychecker.HandleGetQueries) + e.POST("/query", querychecker.HandleAddQuery) + e.PUT("/query", querychecker.HandleUpdateQuery) + e.DELETE("/query", querychecker.HandleDeleteQuery) + + // Start the metrics server + wg.Add(1) + go func() { + logger.Info("starting metrics serverd", "port", cmd.Int("port")) + if err := e.Start(fmt.Sprintf(":%d", cmd.Int("port"))); err != nil { + logger.Error("failed to start metrics server", "err", err) + } + wg.Done() + }() + + select { + case <-signals: + cancel() + fmt.Println("shutting down on signal") + case <-ctx.Done(): + fmt.Println("shutting down on context done") + } + + logger.Info("shutting down, waiting for workers to clean up") + + if err := e.Shutdown(ctx); err != nil { + logger.Error("failed to shut down metrics server", "err", err) + wg.Done() + } + + querychecker.Stop() + + wg.Wait() + logger.Info("shut down successfully") + + return nil +} diff --git a/cmd/rainbow/Dockerfile b/cmd/rainbow/Dockerfile new file mode 100644 index 000000000..1d5e5dd3a --- /dev/null +++ b/cmd/rainbow/Dockerfile @@ -0,0 +1,43 @@ +FROM golang:1.25-bookworm AS build-env + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC +ENV GODEBUG="netdns=go" +ENV GOOS="linux" +ENV GOARCH="amd64" +ENV CGO_ENABLED="1" + +WORKDIR /usr/src/rainbow + +COPY . . + +RUN go mod download && \ + go mod verify + +RUN go build \ + -v \ + -trimpath \ + -tags timetzdata \ + -o /rainbow-bin \ + ./cmd/rainbow + +FROM debian:bookworm-slim + +ENV DEBIAN_FRONTEND="noninteractive" +ENV TZ=Etc/UTC +ENV GODEBUG="netdns=go" + +RUN apt-get update && apt-get install --yes \ + dumb-init \ + ca-certificates \ + runit + +WORKDIR /rainbow +COPY --from=build-env /rainbow-bin /usr/bin/rainbow + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] +CMD ["/usr/bin/rainbow"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="rainbow atproto firehose fanout service" +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" diff --git a/cmd/rainbow/README.md b/cmd/rainbow/README.md new file mode 100644 index 000000000..197a67492 --- /dev/null +++ b/cmd/rainbow/README.md @@ -0,0 +1,33 @@ + +`rainbow`: atproto Firehose Fanout Service +========================================== + +This is an atproto service which consumes from a firehose (eg, from a relay or PDS) and fans out events to many subscribers. + +Features and design points: + +- retains "backfill window" on local disk (using [pebble](https://github.com/cockroachdb/pebble)) +- serves the `com.atproto.sync.subscribeRepos` endpoint (WebSocket) +- proxies through public and administrative API requests to the backing host +- retains upstream firehose "sequence numbers" +- does not validate events (signatures, repo tree, hashes, etc), just passes through +- does not archive or mirror individual records or entire repositories (or implement related API endpoints) +- somewhat disk I/O intensive: fast NVMe disks are recommended, and RAM is helpful for caching +- single golang binary for easy deployment +- observability: logging, prometheus metrics, and OTEL traces + +## Running + +This is a simple, single-binary Go program. You can also build and run it as a docker container (see `./Dockerfile`). + +From the top level of this repo, you can build: + +```shell +go build ./cmd/rainbow -o rainbow +``` + +or just run it, and see configuration options: + +```shell +go run ./cmd/rainbow --help +``` diff --git a/cmd/rainbow/main.go b/cmd/rainbow/main.go new file mode 100644 index 000000000..08b0afb23 --- /dev/null +++ b/cmd/rainbow/main.go @@ -0,0 +1,243 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + _ "github.com/joho/godotenv/autoload" + _ "go.uber.org/automaxprocs" + _ "net/http/pprof" + + "github.com/bluesky-social/indigo/events/pebblepersist" + "github.com/bluesky-social/indigo/splitter" + "github.com/bluesky-social/indigo/util/svcutil" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/urfave/cli/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("exiting", "err", err) + os.Exit(1) + } +} + +func run(args []string) error { + + app := cli.Command{ + Name: "rainbow", + Usage: "atproto firehose fan-out daemon", + Version: versioninfo.Short(), + Action: runSplitter, + } + + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "log-level", + Usage: "log verbosity level (eg: warn, info, debug)", + Sources: cli.EnvVars("RAINBOW_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"), + }, + &cli.StringFlag{ + Name: "upstream-host", + Value: "http://localhost:2470", + Usage: "URL (schema and hostname, no path) of the upstream host (eg, relay)", + Sources: cli.EnvVars("ATP_RELAY_HOST", "RAINBOW_RELAY_HOST"), + }, + &cli.StringFlag{ + Name: "persist-db", + Value: "./rainbow.db", + Usage: "path to persistence db", + Sources: cli.EnvVars("RAINBOW_DB_PATH"), + }, + &cli.StringFlag{ + Name: "cursor-file", + Value: "./rainbow-cursor", + Usage: "write upstream cursor number to this file", + Sources: cli.EnvVars("RAINBOW_CURSOR_PATH"), + }, + &cli.StringFlag{ + Name: "api-listen", + Value: ":2480", + Sources: cli.EnvVars("RAINBOW_API_LISTEN"), + }, + &cli.StringFlag{ + Name: "metrics-listen", + Value: ":2481", + Sources: cli.EnvVars("RAINBOW_METRICS_LISTEN", "SPLITTER_METRICS_LISTEN"), + }, + &cli.Float64Flag{ + Name: "persist-hours", + Value: 24 * 3, + Sources: cli.EnvVars("RAINBOW_PERSIST_HOURS", "SPLITTER_PERSIST_HOURS"), + Usage: "hours to buffer (float, may be fractional)", + }, + &cli.Int64Flag{ + Name: "persist-bytes", + Value: 0, + Usage: "max bytes target for event cache, 0 to disable size target trimming", + Sources: cli.EnvVars("RAINBOW_PERSIST_BYTES", "SPLITTER_PERSIST_BYTES"), + }, + &cli.StringSliceFlag{ + // TODO: better name for this argument + Name: "next-crawler", + Usage: "forward POST requestCrawl to these hosts (schema and host, no path) in addition to upstream-host. Comma-separated or multiple flags", + Sources: cli.EnvVars("RAINBOW_NEXT_CRAWLER", "RELAY_NEXT_CRAWLER"), + }, + &cli.StringFlag{ + Name: "collectiondir-host", + Value: "http://localhost:2510", + Usage: "host (schema and hostname, no path) of upstream collectiondir instance, for com.atproto.sync.listReposByCollection", + Sources: cli.EnvVars("RAINBOW_COLLECTIONDIR_HOST"), + }, + &cli.StringFlag{ + Name: "env", + Usage: "operating environment (eg, 'prod', 'test')", + Value: "dev", + Sources: cli.EnvVars("ENVIRONMENT"), + }, + &cli.BoolFlag{ + Name: "enable-otel-otlp", + Usage: "enables OTEL OTLP exporter endpoint", + Sources: cli.EnvVars("RAINBOW_ENABLE_OTEL_OTLP", "ENABLE_OTEL_OTLP"), + }, + &cli.StringFlag{ + Name: "otel-otlp-endpoint", + Usage: "OTEL traces export endpoint", + Value: "http://localhost:4318", + Sources: cli.EnvVars("OTEL_EXPORTER_OTLP_ENDPOINT"), + }, + } + + return app.Run(context.Background(), args) +} + +func runSplitter(ctx context.Context, cmd *cli.Command) error { + // Trap SIGINT to trigger a shutdown. + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + logger := svcutil.ConfigLogger(cmd, os.Stdout).With("system", "rainbow") + + // Enable OTLP HTTP exporter + // For relevant environment variables: + // https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables + if cmd.Bool("enable-otel-otlp") { + ep := cmd.String("otel-otlp-endpoint") + logger.Info("setting up trace exporter", "endpoint", ep) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + exp, err := otlptracehttp.New(ctx) + if err != nil { + logger.Error("failed to create trace exporter", "error", err) + os.Exit(1) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := exp.Shutdown(ctx); err != nil { + logger.Error("failed to shutdown trace exporter", "error", err) + } + }() + + env := cmd.String("env") + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exp), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("splitter"), + attribute.String("env", env), // DataDog + attribute.String("environment", env), // Others + attribute.Int64("ID", 1), + )), + ) + otel.SetTracerProvider(tp) + } + + persistPath := cmd.String("persist-db") + upstreamHost := cmd.String("upstream-host") + collectionDirHost := cmd.String("collectiondir-host") + nextCrawlers := cmd.StringSlice("next-crawler") + + var spl *splitter.Splitter + var err error + if persistPath != "" { + logger.Info("building splitter with storage at", "path", persistPath) + ppopts := pebblepersist.PebblePersistOptions{ + DbPath: persistPath, + PersistDuration: time.Duration(float64(time.Hour) * cmd.Float64("persist-hours")), + GCPeriod: 5 * time.Minute, + MaxBytes: uint64(cmd.Int64("persist-bytes")), + } + conf := splitter.SplitterConfig{ + UpstreamHost: upstreamHost, + CollectionDirHost: collectionDirHost, + CursorFile: cmd.String("cursor-file"), + PebbleOptions: &ppopts, + UserAgent: fmt.Sprintf("rainbow/%s (atproto-relay)", versioninfo.Short()), + } + spl, err = splitter.NewSplitter(conf, nextCrawlers) + } else { + logger.Info("building in-memory splitter") + conf := splitter.SplitterConfig{ + UpstreamHost: upstreamHost, + CollectionDirHost: collectionDirHost, + CursorFile: cmd.String("cursor-file"), + } + spl, err = splitter.NewSplitter(conf, nextCrawlers) + } + if err != nil { + logger.Error("failed to create splitter", "path", persistPath, "error", err) + os.Exit(1) + return err + } + + // set up metrics endpoint + metricsListen := cmd.String("metrics-listen") + go func() { + if err := spl.StartMetrics(metricsListen); err != nil { + logger.Error("failed to start metrics endpoint", "err", err) + os.Exit(1) + } + }() + + runErr := make(chan error, 1) + + go func() { + err := spl.StartAPI(cmd.String("api-listen")) + runErr <- err + }() + + logger.Info("startup complete") + select { + case <-signals: + logger.Info("received shutdown signal") + if err := spl.Shutdown(); err != nil { + logger.Error("error during Splitter shutdown", "err", err) + } + case err := <-runErr: + if err != nil { + logger.Error("error during Splitter startup", "err", err) + } + logger.Info("shutting down") + if err := spl.Shutdown(); err != nil { + logger.Error("error during Splitter shutdown", "err", err) + } + } + + logger.Info("shutdown complete") + + return nil +} diff --git a/cmd/relay/Dockerfile b/cmd/relay/Dockerfile new file mode 100644 index 000000000..01b27cc8b --- /dev/null +++ b/cmd/relay/Dockerfile @@ -0,0 +1,49 @@ +# Run this dockerfile from the top level of the indigo git repository like: +# +# podman build -f ./cmd/relay/Dockerfile -t relay . + +### Compile stage +FROM golang:1.25-alpine3.22 AS build-env +RUN apk add --no-cache build-base make git + +ADD . /dockerbuild +WORKDIR /dockerbuild + +# timezone data for alpine builds +ENV GOEXPERIMENT=loopvar +RUN GIT_VERSION=$(git describe --tags --long --always) && \ + go build -tags timetzdata -o /relay ./cmd/relay + +### Build Frontend stage +FROM node:18-alpine as web-builder + +WORKDIR /app + +COPY cmd/relay/relay-admin-ui /app/ + +RUN yarn install --frozen-lockfile + +RUN yarn build + +### Run stage +FROM alpine:3.22 + +RUN apk add --no-cache --update dumb-init ca-certificates runit +ENTRYPOINT ["dumb-init", "--"] + +WORKDIR / +RUN mkdir -p data/relay +COPY --from=build-env /relay / +COPY --from=web-builder /app/dist/ public/ + +# small things to make golang binaries work well under alpine +ENV GODEBUG=netdns=go +ENV TZ=Etc/UTC + +EXPOSE 2470 + +CMD ["/relay", "serve"] + +LABEL org.opencontainers.image.source=https://github.com/bluesky-social/indigo +LABEL org.opencontainers.image.description="atproto Relay" +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" diff --git a/cmd/relay/HACKING.md b/cmd/relay/HACKING.md new file mode 100644 index 000000000..5ba140720 --- /dev/null +++ b/cmd/relay/HACKING.md @@ -0,0 +1,57 @@ + + +## Behaviors + +Details about how the relay operates which might not be obvious! + +- unknown/unexpected fields on overall firehose messages (eg, `#commit`) are *not* passed-through, so it is critical to upgrade the relay when there are protocol changes +- records and commit objects *are* passed through verbatim: they are serialized in `blocks` fields on `#commit` and `#sync` messages +- some admin UI changes are persisted across restarts (stored in database), others are not (ephemeral) + - ephemeral (but can be configured via env vars): new-hosts-per-day limit; enable/disable requestCrawl + - persisted (in database): account takedowns, domain bans, host bans, host account limit +- the "lenient mode" configuration flag is intended as a short-term migration tool for [atproto Sync 1.1](https://github.com/bluesky-social/proposals/tree/main/0006-sync-iteration) and will be removed over time +- once an upstream host websocket is established, the sequence numbers on that socket must always increase; messages with lower sequence will be dropped. but this is only strictly enforced over the life the the socket connection; if the relay restarts and the host emits older sequence numbers, those messages will start coming through +- for a new host (no known previous sequence number), the relay will connect at "current" firehose offset, not "oldest" offset and backfill +- for a known host, the relay will attempt to reconnect (eg, after a drop or restart) at the last persisted sequence number. persisting should happen every few seconds, or at clean shutdown of the daemon, but it is possible for the cursor to be slightly out of sync, resulting in replay of messages +- account-level `#commit` revisions must always increase, and these revisions are stored for every valid `#commit` or `#sync` message from the account. repeated or lower revision messages are dropped. messages with revisions corresponding to a TID "in the future" (beyond a fudge period of a few minutes) are also dropped +- messages for an account (DID) which come from a host connection which are not the current PDS host for that account are dropped. If there is a mismatch, the relay will re-resolve the identity (DID document) and double-check before dropping the message, in case there was an account migration not reflected yet in local caches. +- (NOTE: this is not implemented yet!) if a host sends no messages for a long period, the relay should drop the connection and set the host status to "idle"; this is common for low-traffic PDS instances (eg, handful of accounts). The expectation is that the host would then send a `requestCrawl` ping next time there is a new event. +- when the relay restarts, it connects to all "active" hosts +- if configured with "sibling" relay instances, will forward `requestCrawl` and some administrative requests to each of those instances. The use-case is to keep a cluster of independent relays relatively synchronized in terms of hosts subscribed, takedowns, and quotas. Requests are only forwarded if processed successfully on the current instance. `User-Agent` is passed through from original request, but the `Via` header is set, and used to prevent forwarding loops. Auth headers are passed through; admin forwarding only works if the same secret works for all sibling relays. API requests forwarded to a remote rainbow instance (in front of a relay), should get proxied through to that relay successfully. +- both the relay and rainbow set a `Server` header in HTTP responses (including WebSocket connections), and the relay checks for this header when connecting. If it finds the string `atproto-relay` in the header, it refuses the connection, to prevent relay request loops. This is just a conservative default behavior; relays consuming from other relays is allowed by protocol. +- when connecting to remote hosts, including WebSocket subscriptions, the relay includes basic SSRF protections against connecting to private, reserved, or local IP addresses; or ports other than 80 or 443. This check is skipped if the remote host is specifically localhost (with an explicit port). If needed this constraint could be made configurable. +- when connecting to a host (PDS), if the cursor was "in the future", the PDS will return an error frame and drop the connection. In this situation the relay will drop the connection and not automatically reconnect. + + +## Internal Implementation Details + +- the parallel event scheduler prevents multiple tasks for the same account (DID) from being processed at the same time +- note the potential for race-conditions with messages about the same account (DID) coming from different hosts around the same time: in this case there is no guarantee about ordering +- the relay keeps track of which events have been received-but-not-processed by sequence number, and only increments the `lastSeq` for actually-processed events. the "inflight" set of messages (sequence numbers) can grow rather large for active hosts, if there are many events for a single account (only one processed per account at a time) +- the parallel scheduler keeps track of which events have been successfully processed. the slurper event processing code updates the cached sequence after relay processing is done, but this happens within the scheduler work scope, which means when it asks the scheduler what the highest seq is, it will never say the *current* event has been processed. this means lastSeq needs to be pulled periodically, or else is slightly out of date. + + +## Code Organization and History + +*Note: this was written in April 2025, and is likely to get out of date* + +This codebase started as a fork of the prior `bigsky` / "BGS" relay implementation. The host and account state management, and message validation, were re-written. The "slurper" got a refactor, and some event stream and disk persistence code got lighter changes. + +- `Service` struct: overall service executable/daemon. Implements protocol and admin HTTP endpoints. +- `relay.Relay` struct: core relay service logic, message validation and processing, state and database management +- `relay.Slurper` struct: maintains active subscriptions (WebSocket connections) to upstream hosts (eg, PDS instances) +- `relay/models` package: database models +- `stream` package: fork of `indigo:events` package, including websocket "frame" type, listeners, and some event stream rate-limiting +- `stream.XRPCStreamEvent` struct: relatively critical/central serialiation type +- `stream.eventmgr.EventManager`: manages output firehose: disk persistence, sequencing, etc +- `testing` package: end-to-end integration tests + +The `stream` code should probably get merged back in with the `indigo:events` at some point, but there are many small differences so it won't be a quick/trivial change. + + +## Verification Tools and Tests + +- `goat` has several firehose verify flags +- `./testing/` contains a framework for end-to-end relay integration tests +- commit-level MST slice validation tests are in `indigo:atproto/repo` +- there are some interop test resources at: https://github.com/bluesky-social/atproto-interop-tests diff --git a/cmd/relay/README.md b/cmd/relay/README.md new file mode 100644 index 000000000..6c5d280d6 --- /dev/null +++ b/cmd/relay/README.md @@ -0,0 +1,151 @@ + +`relay`: atproto relay reference implementation +=============================================== + +*NOTE: "relays" used to be called "Big Graph Servers", or "BGS", or "bigsky". Many variables and packages still reference "bgs"* + +This is a reference implementation of an atproto relay, written and operated by Bluesky. + +In [atproto](https://atproto.com), a relay subscribes to multiple PDS hosts and outputs a combined "firehose" event stream. Downstream services can subscribe to this single firehose a get all relevant events for the entire network, or a specific sub-graph of the network. The relay verifies repo data structure integrity and identity signatures. It is application-agnostic, and does not validate data records against atproto Lexicon schemas. + +This relay implementation is designed to subscribe to the entire global network. The current state of the codebase is informally expected to scale to around 100 million accounts in the network, and tens of thousands of repo events per second (peak). + +Features and design decisions: + +- runs on a single server (not a distributed system) +- upstream host and account state is stored in a SQL database +- SQL driver: [gorm](https://gorm.io), supporting PostgreSQL in production and sqlite for testing +- highly concurrent: not particularly CPU intensive +- single golang binary for easy deployment +- observability: logging, prometheus metrics, OTEL traces +- admin web interface: configure limits, add upstream PDS instances, etc + +This daemon is relatively simple to self-host, though it isn't as well documented or supported as the PDS reference implementation (see details below). + +See `./HACKING.md` for more documentation of specific behaviors of this implementation. + + +## Development Tips + +The README and Makefile at the top level of this git repo have some generic helpers for testing, linting, formatting code, etc. + +To build the admin web interface, and then build and run the relay locally: + + make build-relay-admin-ui + make run-dev-relay + +You can run the command directly to get a list of configuration flags and environment variables. The environment will be loaded from a `.env`file if one exist: + + go run ./cmd/relay/ --help + +You can also build an run the command directly: + + go build ./cmd/relay + ./relay serve + +By default, the daemon will use sqlite for databases (in the directory `./data/relay/`), and the HTTP API will be bound to localhost port 2470. + +When the daemon isn't running, sqlite database files can be inspected with: + + sqlite3 data/relay/relay.sqlite + [...] + sqlite> .schema + +To wipe all local data (careful!): + + # double-check before running this destructive command + rm -rf ./data/relay/* + +There is a basic web dashboard, though it will not be included unless built and copied to a local directory `./public/`. Run `make build-relay-admin-ui`, and then when running the daemon the dashboard will be available at: . Paste in the admin key, eg `dummy`. + +The local admin routes can also be accessed by passing the admin password using HTTP Basic auth (with username `admin`), for example: + + http get :2470/admin/pds/list -a admin:dummy + +Request crawl of an individual PDS instance like: + + http post :2470/admin/pds/requestCrawl -a admin:dummy hostname=pds.example.com + +The `goat` command line tool includes helpers for administering, inspecting, and debugging relays: + + RELAY_HOST=http://localhost:2470 goat firehose --verify-mst + RELAY_HOST=http://localhost:2470 goat relay admin host list + +## API Endpoints + +This relay implements the core atproto "sync" API endpoints: + +- `GET /xrpc/com.atproto.sync.subscribeRepos` (WebSocket) +- `GET /xrpc/com.atproto.sync.getRepo` (HTTP redirect to account's PDS) +- `GET /xrpc/com.atproto.sync.getRepoStatus` +- `GET /xrpc/com.atproto.sync.listRepos` (optional) +- `GET /xrpc/com.atproto.sync.getLatestCommit` (optional) + +It also implements some relay-specific endpoints: + +- `POST /xrpc/com.atproto.sync.requestCrawl` +- `GET /xrpc/com.atproto.sync.listHosts` +- `GET /xrpc/com.atproto.sync.getHostStatus` + +Documentation can be found in the [atproto specifications](https://atproto.com/specs/sync) for repository synchronization, event streams, data formats, account status, etc. + +This implementation also has some off-protocol admin endpoints under `/admin/`. These have legacy schemas from an earlier implementation, are not well documented, and should not be considered a stable API to build upon. The intention is to refactor them in to Lexicon-specified APIs. + +## Configuration and Operation + +*NOTE: this document is not a complete guide to operating a relay as a public service. That requires planning around acceptable use policies, financial sustainability, infrastructure selection, etc. This is just a quick overview of the mechanics of getting a relay up and running.* + +Some notable configuration env vars: + +- `RELAY_ADMIN_PASSWORD` +- `DATABASE_URL`: eg, `postgres://relay:CHANGEME@localhost:5432/relay` +- `RELAY_PERSIST_DIR`: storage location for "backfill" events, eg `/data/relay/persist` +- `RELAY_REPLAY_WINDOW`: the duration of output "backfill window", eg `24h` +- `RELAY_LENIENT_SYNC_VALIDATION`: if `true`, allow legacy upstreams which don't implement atproto sync v1.1 +- `RELAY_TRUSTED_DOMAINS`: patterns of PDS hosts which get larger quotas by default, eg `*.host.bsky.network` + +There is a health check endpoint at `/xrpc/_health`. Prometheus metrics are exposed by default on port 2471, path `/metrics`. The service logs fairly verbosely to stdout; use `LOG_LEVEL` to control log volume (`warn`, `info`, etc). + +Be sure to double-check bandwidth usage and pricing if running a public relay! Bandwidth prices can vary widely between providers, and popular cloud services (AWS, Google Cloud, Azure) are very expensive compared to alternatives like OVH or Hetzner. + +The relay admin interface has flexibility for many situations, but in some operational incidents it may be necessary to run SQL commands to do cleanups. This should be done when the relay is not actively operating. It is also recommended to run SQL commands in a transaction that can be rolled back in case of a typo or mistake. + +On the public web, you should probably run the relay behind a load-balancer or reverse proxy like `haproxy` or `caddy`, which manages TLS and can have various HTTP limits and behaviors configured. Remember that WebSocket support is required. + +The relay does not resolve atproto handles, but it does do DNS resolutions for hostnames, and may do a burst of resolutions at startup. Note that the go runtime may have an internal DNS implementation enabled (this is the default for the Dockerfile). The relay *will* do a large number of DID resolutions, particularly calls to the PLC directory, and particularly after a process restart when the in-process identity cache is warming up. + +### PostgreSQL + +PostgreSQL is recommended for any non-trival relay deployments. Database configuration is passed via the `DATABASE_URL` environment variable, or the corresponding CLI arg. + +The user and database must already be configured. For example: + + CREATE DATABASE relay; + + CREATE USER ${username} WITH PASSWORD '${password}'; + GRANT ALL PRIVILEGES ON DATABASE relay TO ${username}; + +This service currently uses `gorm` to automatically run database migrations as the regular user. There is no support for running database migrations separately under more privileged database user. + +### Docker + +The relay is relatively easy to build and operate as as simple executable, but there is also Dockerfile in this directory. It can be used to build customized/patched versions of the relay as a container, republish them, run locally, deploy to servers, deploy to an orchestrated cluster, etc. + +Relays process a lot of packets, so we strongly recommend running docker in "host networking" mode when operating a full-network relay. You may also want to use something other than default docker log management (eg, `svlogd`), to handle large log volumes. + +### Bootstrapping Host List + +Before bulk-adding hosts, you should probably increase the "new-hosts-per-day" limit, at least temporarily. + +The relay comes with a helper command to pull a list of hosts from an existing relay. You should shut the relay down first and run this as a separate command: + + ./relay pull-hosts + +An alternative method, using `goat` and `parallel`, which is more gentle and may be better for small servers: + + # dump a host list using goat + # 'rg' is ripgrep + RELAY_HOST=https://relay1.us-west.bsky.network goat relay host list | rg '\tactive' | cut -f1 > hosts.txt + + # assuming that .env contains local relay configuration and admin credential + shuf hosts.txt | parallel goat relay admin host add {} diff --git a/cmd/relay/forward.go b/cmd/relay/forward.go new file mode 100644 index 000000000..d13eb9468 --- /dev/null +++ b/cmd/relay/forward.go @@ -0,0 +1,86 @@ +package main + +import ( + "bytes" + "io" + "net/http" + "strings" + + "github.com/bluesky-social/indigo/cmd/relay/relay" + + "github.com/labstack/echo/v4" +) + +// Forwards HTTP request on to sibling relay instances, if they are configured. +// +// This method expects to be run "in the background" as a goroutine, so it doesn't take a `context.Context`, and does not return an `error`. It logs both success and failure. The `echo.Context` request has presumably been read, so any body is passed separately. The `echo.Context` response is presumably being returned or already finalized concurrently. +func (s *Service) ForwardSiblingRequest(c echo.Context, body []byte) { + + if len(s.config.SiblingRelayHosts) == 0 { + return + } + + // if the request itself was already forwarded, or user-agent seems to be a relay, then don't forward on further (to prevent loops) + req := c.Request() + for _, via := range req.Header.Values("Via") { + if strings.Contains(via, "atproto-relay") { + s.logger.Info("not re-forwarding request to sibling relay", "header", "Via", "value", via) + return + } + } + for _, ua := range req.Header.Values("User-Agent") { + if strings.Contains(ua, "atproto-relay") { + s.logger.Info("not re-forwarding request to sibling relay", "header", "User-Agent", "value", ua) + return + } + } + + for _, rawHost := range s.config.SiblingRelayHosts { + hostname, noSSL, err := relay.ParseHostname(rawHost) + if err != nil { + s.logger.Error("invalid sibling hostname configured", "host", rawHost, "err", err) + return + } + u := req.URL + u.Host = hostname + if noSSL { + u.Scheme = "http" + } else { + u.Scheme = "https" + } + var b io.Reader + if body != nil { + b = bytes.NewBuffer(body) + } + upstreamReq, err := http.NewRequest(req.Method, u.String(), b) + if err != nil { + s.logger.Error("creating admin forward request failed", "method", req.Method, "url", u.String(), "err", err) + continue + } + + // copy some headers from inbound request + for _, hdr := range []string{"Accept", "User-Agent", "Authorization", "Content-Type"} { + val := req.Header.Get(hdr) + if val != "" { + upstreamReq.Header.Set(hdr, val) + } + } + + // add Via header (critical to prevent forwarding loops) + upstreamReq.Header.Add("Via", req.Proto+" atproto-relay") + + upstreamResp, err := s.siblingClient.Do(upstreamReq) + if err != nil { + s.logger.Warn("forwarded admin HTTP request failed", "method", req.Method, "sibling", hostname, "url", u.String(), "err", err) + continue + } + if !(upstreamResp.StatusCode >= 200 && upstreamResp.StatusCode < 300) { + respBytes, _ := io.ReadAll(upstreamResp.Body) + upstreamResp.Body.Close() + s.logger.Warn("forwarded admin HTTP request failed", "method", req.Method, "sibling", hostname, "url", u.String(), "statusCode", upstreamResp.StatusCode, "body", string(respBytes)) + continue + } + upstreamResp.Body.Close() + s.logger.Info("successfully forwarded admin HTTP request", "method", req.Method, "url", u.String()) + } +} diff --git a/cmd/relay/handlers.go b/cmd/relay/handlers.go new file mode 100644 index 000000000..99e318685 --- /dev/null +++ b/cmd/relay/handlers.go @@ -0,0 +1,271 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/cmd/relay/relay" + "github.com/bluesky-social/indigo/cmd/relay/relay/models" + + "github.com/labstack/echo/v4" +) + +func (s *Service) handleComAtprotoSyncRequestCrawl(c echo.Context, body *comatproto.SyncRequestCrawl_Input, admin bool) error { + ctx := c.Request().Context() + + if s.config.DisableRequestCrawl && !admin { + return c.JSON(http.StatusForbidden, atclient.ErrorBody{Name: "Forbidden", Message: "public requestCrawl not allowed on this relay"}) + } + + hostname, noSSL, err := relay.ParseHostname(body.Hostname) + if err != nil { + return c.JSON(http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("hostname field empty or invalid: %s", body.Hostname)}) + } + + if noSSL && !s.config.AllowInsecureHosts && !admin { + return c.JSON(http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "this relay requires host SSL"}) + } + + // TODO: could ensure that query and path are empty + + if strings.HasPrefix(hostname, "localhost:") { + if !admin { + return c.JSON(http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "can not configure localhost via public endpoint"}) + } + // else, allowed + } else { + banned, err := s.relay.DomainIsBanned(ctx, hostname) + if err != nil { + return nil + } + if banned { + return c.JSON(http.StatusUnauthorized, atclient.ErrorBody{Name: "DomainBan", Message: "host domain is banned"}) + } + } + + hostURL := "https://" + hostname + if noSSL { + hostURL = "http://" + hostname + } + + if err := s.relay.HostChecker.CheckHost(ctx, hostURL); err != nil { + return c.JSON(http.StatusBadRequest, atclient.ErrorBody{Name: "HostNotFound", Message: fmt.Sprintf("host server unreachable: %s", err)}) + } + + // forward on to any sibling instances (note that sometimes is, sometimes isn't an admin request) + b, err := json.Marshal(body) + if err != nil { + return err + } + go s.ForwardSiblingRequest(c, b) + + return s.relay.SubscribeToHost(ctx, hostname, noSSL, admin) +} + +func (s *Service) handleComAtprotoSyncListHosts(c echo.Context, cursor int64, limit int) (*comatproto.SyncListHosts_Output, error) { + ctx := c.Request().Context() + + hosts, err := s.relay.ListHosts(ctx, cursor, limit, true) + if err != nil { + return nil, c.JSON(http.StatusInternalServerError, atclient.ErrorBody{Name: "DatabaseError", Message: "failed to list hosts"}) + } + + if len(hosts) == 0 { + // resp.Hosts is an explicit empty array, not just 'nil' + return &comatproto.SyncListHosts_Output{ + Hosts: []*comatproto.SyncListHosts_Host{}, + }, nil + } + + resp := &comatproto.SyncListHosts_Output{ + Hosts: make([]*comatproto.SyncListHosts_Host, len(hosts)), + } + + for i, host := range hosts { + resp.Hosts[i] = &comatproto.SyncListHosts_Host{ + Hostname: host.Hostname, + Seq: &host.LastSeq, + Status: (*string)(&host.Status), + AccountCount: &host.AccountCount, + } + } + + // If this is not the last page, set the cursor + if len(hosts) >= limit && len(hosts) > 1 { + nextCursor := fmt.Sprintf("%d", hosts[len(hosts)-1].ID) + resp.Cursor = &nextCursor + } + + return resp, nil +} + +func (s *Service) handleComAtprotoSyncGetHostStatus(c echo.Context, hostname string) (*comatproto.SyncGetHostStatus_Output, error) { + ctx := c.Request().Context() + + host, err := s.relay.GetHost(ctx, hostname) + if err != nil { + if errors.Is(err, relay.ErrHostNotFound) { + // TODO: test that not found DID is a 404 + return nil, c.JSON(http.StatusNotFound, atclient.ErrorBody{Name: "HostNotFound", Message: "host not found"}) + } + return nil, c.JSON(http.StatusInternalServerError, atclient.ErrorBody{Name: "DatabaseError", Message: "looking up host information"}) + } + + out := &comatproto.SyncGetHostStatus_Output{ + // TODO: AccountCount + Hostname: host.Hostname, + Seq: &host.LastSeq, + Status: (*string)(&host.Status), + } + + return out, nil +} + +func (s *Service) handleComAtprotoSyncListRepos(c echo.Context, cursor int64, limit int) (*comatproto.SyncListRepos_Output, error) { + ctx := c.Request().Context() + + accounts, err := s.relay.ListAccountsDetailed(ctx, cursor, limit) + if err != nil { + s.logger.Error("failed to query accounts", "err", err) + return nil, c.JSON(http.StatusInternalServerError, atclient.ErrorBody{Name: "DatabaseError", Message: "failed to list accounts (repos)"}) + } + + if len(accounts) == 0 { + // resp.Repos is an explicit empty array, not just 'nil' + return &comatproto.SyncListRepos_Output{ + Repos: []*comatproto.SyncListRepos_Repo{}, + }, nil + } + + resp := &comatproto.SyncListRepos_Output{ + Repos: make([]*comatproto.SyncListRepos_Repo, len(accounts)), + } + + for i, acc := range accounts { + active := acc.IsActive() + resp.Repos[i] = &comatproto.SyncListRepos_Repo{ + Did: acc.DID, + Head: acc.CommitCID, + Rev: acc.Rev, + Active: &active, + Status: acc.StatusField(), + } + } + + // If this is not the last page, set the cursor + if len(accounts) >= limit && len(accounts) > 1 { + nextCursor := fmt.Sprintf("%d", accounts[len(accounts)-1].UID) + resp.Cursor = &nextCursor + } + + return resp, nil +} + +func (s *Service) handleComAtprotoSyncGetRepoStatus(c echo.Context, did syntax.DID) (*comatproto.SyncGetRepoStatus_Output, error) { + ctx := c.Request().Context() + + acc, err := s.relay.GetAccount(ctx, did) + if err != nil { + if errors.Is(err, relay.ErrAccountNotFound) { + // TODO: test that not found DID is a 404 + return nil, c.JSON(http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "account not found"}) + } + return nil, c.JSON(http.StatusInternalServerError, atclient.ErrorBody{Name: "DatabaseError", Message: "looking up account information"}) + } + + out := &comatproto.SyncGetRepoStatus_Output{ + Did: did.String(), + Active: acc.IsActive(), + Status: acc.StatusField(), + } + + repo, err := s.relay.GetAccountRepo(ctx, acc.UID) + if err != nil && !errors.Is(err, relay.ErrAccountRepoNotFound) { + return nil, err + } + // ^^ only returns for non-ErrAccountRepoNotFound: repo can be nil after this! + + if repo != nil { + out.Rev = &repo.Rev + } + + return out, nil +} + +func (s *Service) handleComAtprotoSyncGetLatestCommit(c echo.Context, did syntax.DID) (*comatproto.SyncGetLatestCommit_Output, error) { + ctx := c.Request().Context() + + acc, err := s.relay.GetAccount(ctx, did) + if err != nil { + if errors.Is(err, relay.ErrAccountNotFound) { + // TODO: test that not found DID is a 404 + return nil, c.JSON(http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "account not found"}) + } + return nil, c.JSON(http.StatusInternalServerError, atclient.ErrorBody{Name: "DatabaseError", Message: "looking up account information"}) + } + + switch acc.AccountStatus() { + case models.AccountStatusTakendown, models.AccountStatusSuspended: + return nil, c.JSON(http.StatusForbidden, atclient.ErrorBody{Name: "RepoTakendown", Message: "account not active (takendown)"}) + case models.AccountStatusDeactivated: + return nil, c.JSON(http.StatusForbidden, atclient.ErrorBody{Name: "RepoDeactivated", Message: "account not active (deactivated)"}) + case models.AccountStatusDeleted: + return nil, c.JSON(http.StatusForbidden, atclient.ErrorBody{Name: "RepoDeleted", Message: "account not active (deleted)"}) + case models.AccountStatusActive: + // pass + default: + return nil, c.JSON(http.StatusForbidden, atclient.ErrorBody{Name: "RepoInactive", Message: fmt.Sprintf("account not active: %s", acc.AccountStatus())}) + } + + repo, err := s.relay.GetAccountRepo(ctx, acc.UID) + if err != nil { + if errors.Is(err, relay.ErrAccountRepoNotFound) { + return nil, c.JSON(http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotSynchronized", Message: "do not know current repo state for account"}) + } + return nil, err + } + + return &comatproto.SyncGetLatestCommit_Output{ + Cid: repo.CommitCID, + Rev: repo.Rev, + }, nil +} + +type HealthStatus struct { + Status string `json:"status"` + Message string `json:"msg,omitempty"` +} + +func (svc *Service) HandleHealthCheck(c echo.Context) error { + ctx := c.Request().Context() + if err := svc.relay.Healthcheck(ctx); err != nil { + svc.logger.Error("healthcheck can't connect to database", "err", err) + return c.JSON(http.StatusInternalServerError, HealthStatus{Status: "error", Message: "can't connect to database"}) + } else { + return c.JSON(http.StatusOK, HealthStatus{Status: "ok"}) + } +} + +var homeMessage string = ` +.########..########.##..........###....##....## +.##.....##.##.......##.........##.##....##..##. +.##.....##.##.......##........##...##....####.. +.########..######...##.......##.....##....##... +.##...##...##.......##.......#########....##... +.##....##..##.......##.......##.....##....##... +.##.....##.########.########.##.....##....##... + +This is an atproto [https://atproto.com] relay instance, running the 'relay' codebase [https://github.com/bluesky-social/indigo] + +The firehose WebSocket path is at: /xrpc/com.atproto.sync.subscribeRepos +` + +func (svc *Service) HandleHomeMessage(c echo.Context) error { + return c.String(http.StatusOK, homeMessage) +} diff --git a/cmd/relay/handlers_admin.go b/cmd/relay/handlers_admin.go new file mode 100644 index 000000000..3190ee693 --- /dev/null +++ b/cmd/relay/handlers_admin.go @@ -0,0 +1,515 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/cmd/relay/relay" + "github.com/bluesky-social/indigo/cmd/relay/relay/models" + + "github.com/labstack/echo/v4" + dto "github.com/prometheus/client_model/go" +) + +// This endpoint is basically the same as the regular com.atproto.sync.requestCrawl endpoint, except it sets a flag to bypass configuration checks. +func (s *Service) handleAdminRequestCrawl(c echo.Context) error { + var body comatproto.SyncRequestCrawl_Input + if err := c.Bind(&body); err != nil { + return &echo.HTTPError{Code: http.StatusBadRequest, Message: fmt.Sprintf("invalid body: %s", err)} + } + + return s.handleComAtprotoSyncRequestCrawl(c, &body, true) +} + +func (s *Service) handleAdminSetSubsEnabled(c echo.Context) error { + enabled, err := strconv.ParseBool(c.QueryParam("enabled")) + if err != nil { + return &echo.HTTPError{Code: http.StatusBadRequest, Message: err.Error()} + } + s.config.DisableRequestCrawl = !enabled + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +func (s *Service) handleAdminGetSubsEnabled(c echo.Context) error { + return c.JSON(http.StatusOK, map[string]bool{ + "enabled": !s.config.DisableRequestCrawl, + }) +} + +func (s *Service) handleAdminGetNewHostPerDayRateLimit(c echo.Context) error { + return c.JSON(http.StatusOK, map[string]int64{ + "limit": s.relay.HostPerDayLimiter.Limit(), + }) +} + +func (s *Service) handleAdminSetNewHostPerDayRateLimit(c echo.Context) error { + limit, err := strconv.ParseInt(c.QueryParam("limit"), 10, 64) + if err != nil { + return &echo.HTTPError{Code: http.StatusBadRequest, Message: fmt.Errorf("failed to parse limit: %w", err).Error()} + } + + s.relay.HostPerDayLimiter.SetLimit(limit) + + // NOTE: *not* forwarding to sibling instances + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +func (s *Service) handleAdminTakeDownRepo(c echo.Context) error { + ctx := c.Request().Context() + + var body map[string]string + if err := c.Bind(&body); err != nil { + return err + } + didField, ok := body["did"] + if !ok { + return &echo.HTTPError{ + Code: http.StatusBadRequest, + Message: "must specify DID parameter in body", + } + } + did, err := syntax.ParseDID(didField) + if err != nil { + return err + } + + if err := s.relay.UpdateAccountLocalStatus(ctx, did, models.AccountStatusTakendown, true); err != nil { + if errors.Is(err, relay.ErrAccountNotFound) { + return &echo.HTTPError{ + Code: http.StatusNotFound, + Message: "account not found", + } + } + return &echo.HTTPError{ + Code: http.StatusInternalServerError, + Message: err.Error(), + } + } + + // forward on to any sibling instances + b, err := json.Marshal(body) + if err != nil { + return err + } + go s.ForwardSiblingRequest(c, b) + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +func (s *Service) handleAdminReverseTakedown(c echo.Context) error { + ctx := c.Request().Context() + + var body map[string]string + if err := c.Bind(&body); err != nil { + return err + } + didField, ok := body["did"] + if !ok { + return &echo.HTTPError{ + Code: http.StatusBadRequest, + Message: "must specify DID parameter in body", + } + } + did, err := syntax.ParseDID(didField) + if err != nil { + return err + } + + if err := s.relay.UpdateAccountLocalStatus(ctx, did, models.AccountStatusActive, true); err != nil { + if errors.Is(err, relay.ErrAccountNotFound) { + return &echo.HTTPError{ + Code: http.StatusNotFound, + Message: "repo not found", + } + } + return &echo.HTTPError{ + Code: http.StatusInternalServerError, + Message: err.Error(), + } + } + + // forward on to any sibling instances + b, err := json.Marshal(body) + if err != nil { + return err + } + go s.ForwardSiblingRequest(c, b) + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +type ListTakedownsResponse struct { + DIDs []string `json:"dids"` + Cursor int64 `json:"cursor,omitempty"` +} + +func (s *Service) handleAdminListRepoTakeDowns(c echo.Context) error { + ctx := c.Request().Context() + var err error + + limit := 500 + cursor := int64(0) + cursorQuery := c.QueryParam("cursor") + if cursorQuery != "" { + cursor, err = strconv.ParseInt(cursorQuery, 10, 64) + if err != nil { + return &echo.HTTPError{Code: http.StatusBadRequest, Message: "invalid cursor param"} + } + } + + accounts, err := s.relay.ListAccountTakedowns(ctx, cursor, limit) + if err != nil { + return &echo.HTTPError{Code: http.StatusInternalServerError, Message: "failed to list takedowns"} + } + + out := ListTakedownsResponse{ + DIDs: make([]string, len(accounts)), + } + for i, acc := range accounts { + out.DIDs[i] = acc.DID + out.Cursor = int64(acc.UID) + } + if len(out.DIDs) < limit { + out.Cursor = 0 + } + return c.JSON(http.StatusOK, out) +} + +func (s *Service) handleAdminGetUpstreamConns(c echo.Context) error { + return c.JSON(http.StatusOK, s.relay.Slurper.GetActiveSubHostnames()) +} + +type rateLimit struct { + Max float64 `json:"Max"` + WindowSeconds float64 `json:"Window"` +} + +type hostInfo struct { + // fields from old models.PDS + ID uint64 + CreatedAt time.Time + Host string + SSL bool + Cursor int64 + Registered bool + Blocked bool + CrawlRateLimit float64 + RepoCount int64 + RepoLimit int64 + + HasActiveConnection bool `json:"HasActiveConnection"` + EventsSeenSinceStartup uint64 `json:"EventsSeenSinceStartup"` + PerSecondEventRate rateLimit `json:"PerSecondEventRate"` + PerHourEventRate rateLimit `json:"PerHourEventRate"` + PerDayEventRate rateLimit `json:"PerDayEventRate"` + UserCount int64 `json:"UserCount"` +} + +func (s *Service) handleListHosts(c echo.Context) error { + ctx := c.Request().Context() + + limit := 10_000 + hosts, err := s.relay.ListHosts(ctx, 0, limit, false) + if err != nil { + return err + } + + activeHostnames := s.relay.Slurper.GetActiveSubHostnames() + activeHosts := make(map[string]bool, len(activeHostnames)) + for _, hostname := range activeHostnames { + activeHosts[hostname] = true + } + + hostInfos := make([]hostInfo, len(hosts)) + for i, host := range hosts { + _, isActive := activeHosts[host.Hostname] + hostInfos[i] = hostInfo{ + ID: host.ID, + CreatedAt: host.CreatedAt, + Host: host.Hostname, + SSL: !host.NoSSL, + Cursor: host.LastSeq, + Registered: host.Status == models.HostStatusActive, // is this right? + Blocked: host.Status == models.HostStatusBanned, + RepoCount: host.AccountCount, + RepoLimit: host.AccountLimit, + + HasActiveConnection: isActive, + UserCount: host.AccountCount, + } + + // fetch current rate limits + hostInfos[i].PerSecondEventRate = rateLimit{Max: -1.0, WindowSeconds: 1} + hostInfos[i].PerHourEventRate = rateLimit{Max: -1.0, WindowSeconds: 3600} + hostInfos[i].PerDayEventRate = rateLimit{Max: -1.0, WindowSeconds: 86400} + if isActive { + slc, err := s.relay.Slurper.GetLimits(host.Hostname) + if err != nil { + s.logger.Error("fetching subscribed host limits", "err", err) + } else { + hostInfos[i].PerSecondEventRate = rateLimit{ + Max: float64(slc.PerSecond), + WindowSeconds: 1, + } + hostInfos[i].PerHourEventRate = rateLimit{ + Max: float64(slc.PerHour), + WindowSeconds: 3600, + } + hostInfos[i].PerDayEventRate = rateLimit{ + Max: float64(slc.PerDay), + WindowSeconds: 86400, + } + } + } + + // pull event counter metrics from prometheus + var m = &dto.Metric{} + if err := relay.EventsReceivedCounter.WithLabelValues(host.Hostname).Write(m); err != nil { + hostInfos[i].EventsSeenSinceStartup = 0 + continue + } + hostInfos[i].EventsSeenSinceStartup = uint64(m.Counter.GetValue()) + } + + return c.JSON(http.StatusOK, hostInfos) +} + +func (s *Service) handleAdminListConsumers(c echo.Context) error { + return c.JSON(http.StatusOK, s.relay.ListConsumers()) +} + +func (s *Service) handleAdminKillUpstreamConn(c echo.Context) error { + ctx := c.Request().Context() + + queryHost := strings.TrimSpace(c.QueryParam("host")) + hostname, _, err := relay.ParseHostname(queryHost) + if err != nil { + return &echo.HTTPError{ + Code: http.StatusBadRequest, + Message: "must pass a valid host", + } + } + + banHost := strings.ToLower(c.QueryParam("block")) == "true" + + // TODO: move this method to relay (for updating the database) + if err := s.relay.Slurper.KillUpstreamConnection(ctx, hostname, banHost); err != nil { + if errors.Is(err, relay.ErrHostInactive) { + return &echo.HTTPError{ + Code: http.StatusBadRequest, + Message: "no active connection to given host", + } + } + return err + } + + // forward on to any sibling instances + go s.ForwardSiblingRequest(c, nil) + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +func (s *Service) handleBlockHost(c echo.Context) error { + ctx := c.Request().Context() + + queryHost := strings.TrimSpace(c.QueryParam("host")) + hostname, _, err := relay.ParseHostname(queryHost) + if err != nil { + return &echo.HTTPError{ + Code: http.StatusBadRequest, + Message: "must pass a valid hostname", + } + } + + host, err := s.relay.GetHost(ctx, hostname) + if err != nil { + return err + } + + if host.Status != models.HostStatusBanned { + if err := s.relay.UpdateHostStatus(ctx, host.ID, models.HostStatusBanned); err != nil { + return err + } + } + + // kill any active connection (there may not be one, so ignore error) + _ = s.relay.Slurper.KillUpstreamConnection(ctx, host.Hostname, false) + + // forward on to any sibling instances + go s.ForwardSiblingRequest(c, nil) + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +func (s *Service) handleUnblockHost(c echo.Context) error { + ctx := c.Request().Context() + + queryHost := strings.TrimSpace(c.QueryParam("host")) + hostname, _, err := relay.ParseHostname(queryHost) + if err != nil { + return &echo.HTTPError{ + Code: http.StatusBadRequest, + Message: "must pass a valid hostname", + } + } + + host, err := s.relay.GetHost(ctx, hostname) + if err != nil { + return err + } + + if host.Status != models.HostStatusActive { + if err := s.relay.UpdateHostStatus(ctx, host.ID, models.HostStatusActive); err != nil { + return err + } + } + + // forward on to any sibling instances + go s.ForwardSiblingRequest(c, nil) + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +type bannedDomains struct { + BannedDomains []string `json:"banned_domains"` +} + +func (s *Service) handleAdminListDomainBans(c echo.Context) error { + ctx := c.Request().Context() + + bans, err := s.relay.ListDomainBans(ctx) + if err != nil { + return err + } + + resp := bannedDomains{ + BannedDomains: make([]string, len(bans)), + } + + for i, ban := range bans { + resp.BannedDomains[i] = ban.Domain + } + + return c.JSON(http.StatusOK, resp) +} + +type banDomainBody struct { + Domain string +} + +func (s *Service) handleAdminBanDomain(c echo.Context) error { + ctx := c.Request().Context() + + var body banDomainBody + if err := c.Bind(&body); err != nil { + return err + } + + err := s.relay.CreateDomainBan(ctx, body.Domain) + if err != nil { + return err + } + + // forward on to any sibling instances + b, err := json.Marshal(body) + if err != nil { + return err + } + go s.ForwardSiblingRequest(c, b) + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +func (s *Service) handleAdminUnbanDomain(c echo.Context) error { + ctx := c.Request().Context() + + var body banDomainBody + if err := c.Bind(&body); err != nil { + return err + } + + err := s.relay.RemoveDomainBan(ctx, body.Domain) + if err != nil { + return err + } + + // forward on to any sibling instances + b, err := json.Marshal(body) + if err != nil { + return err + } + go s.ForwardSiblingRequest(c, b) + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} + +type RateLimitChangeRequest struct { + Hostname string `json:"host"` + RepoLimit *int64 `json:"repo_limit"` +} + +func (s *Service) handleAdminChangeHostRateLimits(c echo.Context) error { + ctx := c.Request().Context() + + var body RateLimitChangeRequest + if err := c.Bind(&body); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid body: %s", err)) + } + + hostname, _, err := relay.ParseHostname(body.Hostname) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("invalid hostname: %s", err)) + } + + // catch empty/nil body + if body.RepoLimit == nil { + return echo.NewHTTPError(http.StatusBadRequest, "missing repo_limit parameter") + } + + host, err := s.relay.GetHost(ctx, hostname) + if err != nil { + // TODO: technically, there could be a database error here or something + return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("unknown hostname: %s", err)) + } + + if err := s.relay.UpdateHostAccountLimit(ctx, host.ID, *body.RepoLimit); err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("failed to update limits: %s", err)) + } + + // forward on to any sibling instances + b, err := json.Marshal(body) + if err != nil { + return err + } + go s.ForwardSiblingRequest(c, b) + + return c.JSON(http.StatusOK, map[string]any{ + "success": "true", + }) +} diff --git a/cmd/relay/main.go b/cmd/relay/main.go new file mode 100644 index 000000000..9ff35a07a --- /dev/null +++ b/cmd/relay/main.go @@ -0,0 +1,342 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "log/slog" + "os" + "os/signal" + "strings" + "syscall" + "time" + + _ "net/http/pprof" + + _ "github.com/joho/godotenv/autoload" + _ "go.uber.org/automaxprocs" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/cmd/relay/relay" + "github.com/bluesky-social/indigo/cmd/relay/stream/eventmgr" + "github.com/bluesky-social/indigo/cmd/relay/stream/persist/diskpersist" + "github.com/bluesky-social/indigo/util/cliutil" + + "github.com/earthboundkid/versioninfo/v2" + "github.com/urfave/cli/v3" + "gorm.io/plugin/opentelemetry/tracing" +) + +func main() { + if err := run(os.Args); err != nil { + slog.Error("exiting process", "err", err.Error()) + os.Exit(-1) + } +} + +func run(args []string) error { + + app := cli.Command{ + Name: "relay", + Usage: "atproto relay daemon", + Version: versioninfo.Short(), + } + app.Flags = []cli.Flag{ + &cli.StringSliceFlag{ + Name: "admin-password", + Usage: "secret password/token for accessing admin endpoints (multiple values allowed)", + Sources: cli.EnvVars("RELAY_ADMIN_PASSWORD", "RELAY_ADMIN_KEY"), + }, + &cli.StringFlag{ + Name: "plc-host", + Usage: "method, hostname, and port of PLC registry", + Value: "https://plc.directory", + Sources: cli.EnvVars("RELAY_PLC_HOST", "ATP_PLC_HOST"), + }, + &cli.StringFlag{ + Name: "log-level", + Usage: "log verbosity level (eg: warn, info, debug)", + Sources: cli.EnvVars("RELAY_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"), + }, + } + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "serve", + Usage: "run the relay daemon", + Action: runRelay, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "db-url", + Usage: "database connection string for relay database", + Value: "sqlite://data/relay/relay.sqlite", + Sources: cli.EnvVars("DATABASE_URL"), + }, + &cli.IntFlag{ + Name: "max-db-conn", + Usage: "limit on size of database connection pool", + Sources: cli.EnvVars("MAX_DB_CONNECTIONS", "MAX_METADB_CONNECTIONS"), + Value: 40, + }, + &cli.StringFlag{ + Name: "bind", + Usage: "IP or address, and port, to listen on for HTTP APIs (including firehose)", + Value: ":2470", + Sources: cli.EnvVars("RELAY_API_BIND", "RELAY_API_LISTEN"), + }, + &cli.StringFlag{ + Name: "persist-dir", + Usage: "local folder to store firehose playback files", + Value: "data/relay/persist", + Sources: cli.EnvVars("RELAY_PERSIST_DIR", "RELAY_PERSISTER_DIR"), + }, + &cli.DurationFlag{ + Name: "replay-window", + Usage: "retention duration for firehose playback", + Sources: cli.EnvVars("RELAY_REPLAY_WINDOW", "RELAY_EVENT_PLAYBACK_TTL"), + Value: 72 * time.Hour, + }, + &cli.IntFlag{ + Name: "host-concurrency", + Usage: "number of concurrent worker routines per upstream host", + Sources: cli.EnvVars("RELAY_HOST_CONCURRENCY", "RELAY_CONCURRENCY_PER_PDS"), + Value: 40, + }, + &cli.Int64Flag{ + Name: "default-account-limit", + Value: 100, + Usage: "max number of active accounts for new upstream hosts", + Sources: cli.EnvVars("RELAY_DEFAULT_ACCOUNT_LIMIT", "RELAY_DEFAULT_REPO_LIMIT"), + }, + &cli.Int64Flag{ + Name: "new-hosts-per-day-limit", + Value: 50, + Usage: "max number of new upstream hosts subscribed per day via public requestCrawl", + Sources: cli.EnvVars("RELAY_NEW_HOSTS_PER_DAY_LIMIT"), + }, + &cli.IntFlag{ + Name: "ident-cache-size", + Value: 5_000_000, + Usage: "size of in-process identity cache (eg, DID docs)", + Sources: cli.EnvVars("RELAY_IDENT_CACHE_SIZE", "RELAY_DID_CACHE_SIZE"), + }, + &cli.BoolFlag{ + Name: "disable-request-crawl", + Usage: "don't process public (un-authenticated) com.atproto.sync.requestCrawl", + Sources: cli.EnvVars("RELAY_DISABLE_REQUEST_CRAWL"), + }, + &cli.BoolFlag{ + Name: "allow-insecure-hosts", + Usage: "enables subscription to non-SSL hosts via requestCrawl", + Sources: cli.EnvVars("RELAY_ALLOW_INSECURE_HOSTS"), + }, + &cli.BoolFlag{ + Name: "lenient-sync-validation", + Usage: "when messages fail atproto 'Sync 1.1' validation, just log, don't drop", + Sources: cli.EnvVars("RELAY_LENIENT_SYNC_VALIDATION"), + }, + &cli.Int64Flag{ + Name: "initial-seq-number", + Usage: "output firehose seq will be greater or equal than this number (will jump ahead if needed)", + Value: 1, + Sources: cli.EnvVars("RELAY_INITIAL_SEQ_NUMBER"), + }, + &cli.StringSliceFlag{ + Name: "sibling-relays", + Usage: "servers (eg https://example.com) to forward admin state changes to; multiple allowed", + Sources: cli.EnvVars("RELAY_SIBLING_RELAYS"), + }, + &cli.StringSliceFlag{ + Name: "trusted-domains", + Usage: "domain names which mark trusted hosts; use wildcard prefix to match suffixes", + Value: []string{"*.host.bsky.network"}, + Sources: cli.EnvVars("RELAY_TRUSTED_DOMAINS"), + }, + &cli.StringFlag{ + Name: "env", + Value: "dev", + Sources: cli.EnvVars("ENVIRONMENT"), + Usage: "declared hosting environment (prod, qa, etc); used in metrics", + }, + &cli.BoolFlag{ + Name: "enable-db-tracing", + }, + &cli.BoolFlag{ + Name: "enable-jaeger-tracing", + }, + &cli.BoolFlag{ + Name: "enable-otel-tracing", + }, + &cli.StringFlag{ + Name: "metrics-listen", + Usage: "IP or address, and port, to listen on for prometheus metrics", + Value: ":2471", + Sources: cli.EnvVars("RELAY_METRICS_LISTEN"), + }, + &cli.StringFlag{ + Name: "otel-exporter-otlp-endpoint", + Value: "http://localhost:4328", + Sources: cli.EnvVars("OTEL_EXPORTER_OTLP_ENDPOINT"), + }, + }, + }, + // additional commands defined in pull.go + cmdPullHosts, + } + return app.Run(context.Background(), args) + +} + +func configLogger(cmd *cli.Command, writer io.Writer) *slog.Logger { + var level slog.Level + switch strings.ToLower(cmd.String("log-level")) { + case "error": + level = slog.LevelError + case "warn": + level = slog.LevelWarn + case "info": + level = slog.LevelInfo + case "debug": + level = slog.LevelDebug + default: + level = slog.LevelInfo + } + logger := slog.New(slog.NewJSONHandler(writer, &slog.HandlerOptions{ + Level: level, + })) + slog.SetDefault(logger) + return logger +} + +func runRelay(ctx context.Context, cmd *cli.Command) error { + logger := configLogger(cmd, os.Stdout) + + // Trap SIGINT to trigger a shutdown. + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + dburl := cmd.String("db-url") + maxConn := cmd.Int("max-db-conn") + logger.Info("configuring database", "url", dburl, "maxConn", maxConn) + db, err := cliutil.SetupDatabase(dburl, maxConn) + if err != nil { + return err + } + + // TODO: add shared external cache + baseDir := identity.BaseDirectory{ + SkipHandleVerification: true, + SkipDNSDomainSuffixes: []string{".bsky.social"}, + TryAuthoritativeDNS: true, + PLCURL: cmd.String("plc-host"), + } + dir := identity.NewCacheDirectory(&baseDir, cmd.Int("ident-cache-size"), time.Hour*24, time.Minute*2, time.Minute*5) + + persistDir := cmd.String("persist-dir") + if err := os.MkdirAll(persistDir, os.ModePerm); err != nil { + return err + } + persitConfig := diskpersist.DefaultDiskPersistOptions() + persitConfig.Retention = cmd.Duration("replay-window") + persitConfig.InitialSeq = cmd.Int64("initial-seq-number") + if persitConfig.InitialSeq <= 0 { + // belt-and-suspenders: the disk persister also checks this internally + return fmt.Errorf("negative or zero initial sequence config: %d", persitConfig.InitialSeq) + } + logger.Info("setting up disk persister", "dir", persistDir, "replayWindow", persitConfig.Retention, "initialSeq", persitConfig.InitialSeq) + persister, err := diskpersist.NewDiskPersistence(persistDir, "", db, persitConfig) + if err != nil { + return fmt.Errorf("setting up disk persister: %w", err) + } + + relayConfig := relay.DefaultRelayConfig() + relayConfig.UserAgent = fmt.Sprintf("indigo-relay/%s (atproto-relay)", versioninfo.Short()) + relayConfig.ConcurrencyPerHost = cmd.Int("host-concurrency") + relayConfig.DefaultRepoLimit = cmd.Int64("default-account-limit") + relayConfig.HostPerDayLimit = cmd.Int64("new-hosts-per-day-limit") + relayConfig.TrustedDomains = cmd.StringSlice("trusted-domains") + relayConfig.LenientSyncValidation = cmd.Bool("lenient-sync-validation") + + svcConfig := DefaultServiceConfig() + svcConfig.AllowInsecureHosts = cmd.Bool("allow-insecure-hosts") + svcConfig.DisableRequestCrawl = cmd.Bool("disable-request-crawl") + svcConfig.SiblingRelayHosts = cmd.StringSlice("sibling-relays") + if len(svcConfig.SiblingRelayHosts) > 0 { + logger.Info("sibling relay hosts configured for admin state forwarding", "servers", svcConfig.SiblingRelayHosts) + } + if cmd.IsSet("admin-password") { + svcConfig.AdminPasswords = cmd.StringSlice("admin-password") + } else { + var rblob [10]byte + _, _ = rand.Read(rblob[:]) + randPassword := base64.URLEncoding.EncodeToString(rblob[:]) + svcConfig.AdminPasswords = []string{randPassword} + logger.Info("generated random admin password", "username", "admin", "password", randPassword) + } + + evtman := eventmgr.NewEventManager(persister) + + logger.Info("constructing relay service") + r, err := relay.NewRelay(db, evtman, &dir, relayConfig) + if err != nil { + return err + } + svc, err := NewService(r, svcConfig) + if err != nil { + return err + } + persister.SetUidSource(r) + + // start metrics endpoint + go func() { + if err := svc.StartMetrics(cmd.String("metrics-listen")); err != nil { + logger.Error("failed to start metrics endpoint", "err", err) + os.Exit(1) + } + }() + + // start observability/tracing (OTEL and jaeger) + if err := setupOTEL(cmd); err != nil { + return err + } + if cmd.Bool("enable-db-tracing") { + if err := db.Use(tracing.NewPlugin()); err != nil { + return err + } + } + + // restart any existing subscriptions as worker goroutines + if err := r.ResubscribeAllHosts(ctx); err != nil { + return err + } + + svcErr := make(chan error, 1) + go func() { + err := svc.StartAPI(cmd.String("bind")) + svcErr <- err + }() + + logger.Info("startup complete") + select { + case <-signals: + logger.Info("received shutdown signal") + errs := svc.Shutdown() + for err := range errs { + logger.Error("error during shutdown", "err", err) + } + case err := <-svcErr: + if err != nil { + logger.Error("error during startup", "err", err) + } + logger.Info("shutting down") + errs := svc.Shutdown() + for err := range errs { + logger.Error("error during shutdown", "err", err) + } + } + + logger.Info("shutdown complete") + + return nil +} diff --git a/cmd/relay/otel.go b/cmd/relay/otel.go new file mode 100644 index 000000000..e1e710715 --- /dev/null +++ b/cmd/relay/otel.go @@ -0,0 +1,86 @@ +package main + +import ( + "context" + "log/slog" + "os" + "time" + + "github.com/urfave/cli/v3" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" +) + +func setupOTEL(cmd *cli.Command) error { + + env := cmd.String("env") + if env == "" { + env = "dev" + } + + if cmd.Bool("enable-jaeger-tracing") { + jaegerUrl := "http://localhost:14268/api/traces" + exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerUrl))) + if err != nil { + return err + } + tp := tracesdk.NewTracerProvider( + // Always be sure to batch in production. + tracesdk.WithBatcher(exp), + // Record information about this application in a Resource. + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("relay"), + attribute.String("env", env), // DataDog + attribute.String("environment", env), // Others + attribute.Int64("ID", 1), + )), + ) + + otel.SetTracerProvider(tp) + } + + // Enable OTLP HTTP exporter + // For relevant environment variables: + // https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables + // At a minimum, you need to set + // OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 + if cmd.Bool("enable-otel-tracing") { + ep := cmd.String("otel-exporter-otlp-endpoint") + slog.Info("setting up trace exporter", "endpoint", ep) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + exp, err := otlptracehttp.New(ctx) + if err != nil { + slog.Error("failed to create trace exporter", "error", err) + os.Exit(1) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := exp.Shutdown(ctx); err != nil { + slog.Error("failed to shutdown trace exporter", "error", err) + } + }() + + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exp), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("relay"), + attribute.String("env", env), // DataDog + attribute.String("environment", env), // Others + attribute.Int64("ID", 1), + )), + ) + otel.SetTracerProvider(tp) + } + + return nil +} diff --git a/cmd/relay/pull.go b/cmd/relay/pull.go new file mode 100644 index 000000000..3f917857d --- /dev/null +++ b/cmd/relay/pull.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "errors" + "fmt" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/cmd/relay/relay" + "github.com/bluesky-social/indigo/cmd/relay/relay/models" + "github.com/bluesky-social/indigo/util/cliutil" + + "github.com/urfave/cli/v3" +) + +var cmdPullHosts = &cli.Command{ + Name: "pull-hosts", + Usage: "initializes or updates host list from an existing relay (public API)", + Action: runPullHosts, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "relay-host", + Usage: "method, hostname, and port of relay to pull from", + Value: "https://bsky.network", + Sources: cli.EnvVars("RELAY_HOST"), + }, + &cli.StringFlag{ + Name: "db-url", + Usage: "database connection string for relay database", + Value: "sqlite://data/relay/relay.sqlite", + Sources: cli.EnvVars("DATABASE_URL"), + }, + &cli.Int64Flag{ + Name: "default-account-limit", + Value: 100, + Usage: "max number of active accounts for new upstream hosts", + Sources: cli.EnvVars("RELAY_DEFAULT_ACCOUNT_LIMIT", "RELAY_DEFAULT_REPO_LIMIT"), + }, + &cli.Int64Flag{ + Name: "batch-size", + Value: 500, + Usage: "host many hosts to pull at a time", + Sources: cli.EnvVars("RELAY_PULL_HOSTS_BATCH_SIZE"), + }, + &cli.StringSliceFlag{ + Name: "trusted-domains", + Usage: "domain names which mark trusted hosts; use wildcard prefix to match suffixes", + Value: []string{"*.host.bsky.network"}, + Sources: cli.EnvVars("RELAY_TRUSTED_DOMAINS"), + }, + &cli.BoolFlag{ + Name: "skip-host-checks", + Usage: "don't run describeServer requests to see if host is a PDS before adding", + Sources: cli.EnvVars("RELAY_SKIP_HOST_CHECKS"), + }, + }, +} + +func runPullHosts(ctx context.Context, cmd *cli.Command) error { + + if cmd.Args().Len() > 0 { + return fmt.Errorf("unexpected arguments") + } + + client := atclient.NewAPIClient(cmd.String("relay-host")) + + skipHostChecks := cmd.Bool("skip-host-checks") + + dir := identity.DefaultDirectory() + + dburl := cmd.String("db-url") + db, err := cliutil.SetupDatabase(dburl, 10) + if err != nil { + return err + } + + relayConfig := relay.DefaultRelayConfig() + relayConfig.DefaultRepoLimit = cmd.Int64("default-account-limit") + relayConfig.TrustedDomains = cmd.StringSlice("trusted-domains") + + // NOTE: setting evtmgr to nil + r, err := relay.NewRelay(db, nil, dir, relayConfig) + if err != nil { + return err + } + + checker := relay.NewHostClient(relayConfig.UserAgent) + + cursor := "" + size := cmd.Int64("batch-size") + for { + resp, err := comatproto.SyncListHosts(ctx, client, cursor, size) + if err != nil { + return err + } + for _, h := range resp.Hosts { + if h.Status == nil { + fmt.Printf("%s: status=unknown\n", h.Hostname) + continue + } + if !(models.HostStatus(*h.Status) == models.HostStatusActive || models.HostStatus(*h.Status) == models.HostStatusIdle) { + fmt.Printf("%s: status=%s\n", h.Hostname, *h.Status) + continue + } + if h.Seq == nil || *h.Seq <= 0 { + fmt.Printf("%s: no-cursor\n", h.Hostname) + continue + } + existing, err := r.GetHost(ctx, h.Hostname) + if err != nil && !errors.Is(err, relay.ErrHostNotFound) { + return err + } + if existing != nil { + fmt.Printf("%s: exists\n", h.Hostname) + continue + } + hostname, noSSL, err := relay.ParseHostname(h.Hostname) + if err != nil { + return fmt.Errorf("%w: %s", err, h.Hostname) + } + if noSSL { + // skip "localhost" and non-SSL hosts (this is for public PDS instances) + fmt.Printf("%s: non-public\n", h.Hostname) + continue + } + + accountLimit := r.Config.DefaultRepoLimit + trusted := relay.IsTrustedHostname(hostname, r.Config.TrustedDomains) + if trusted { + accountLimit = r.Config.TrustedRepoLimit + } + + if !skipHostChecks { + if err := checker.CheckHost(ctx, "https://"+hostname); err != nil { + fmt.Printf("%s: checking host: %s\n", h.Hostname, err) + continue + } + } + + host := models.Host{ + Hostname: hostname, + NoSSL: noSSL, + Status: models.HostStatusActive, + Trusted: trusted, + AccountLimit: accountLimit, + } + if err := db.Create(&host).Error; err != nil { + return err + } + fmt.Printf("%s: added\n", h.Hostname) + } + if resp.Cursor == nil || *resp.Cursor == "" { + break + } + cursor = *resp.Cursor + } + return nil +} diff --git a/cmd/relay/relay-admin-ui/.gitignore b/cmd/relay/relay-admin-ui/.gitignore new file mode 100644 index 000000000..1a993212d --- /dev/null +++ b/cmd/relay/relay-admin-ui/.gitignore @@ -0,0 +1,3 @@ +dist/ +node_modules/ +package-lock.json diff --git a/cmd/relay/relay-admin-ui/index.html b/cmd/relay/relay-admin-ui/index.html new file mode 100644 index 000000000..5c5a443e6 --- /dev/null +++ b/cmd/relay/relay-admin-ui/index.html @@ -0,0 +1,13 @@ + + + + + + + Relay Dashboard + + +
+ + + diff --git a/cmd/relay/relay-admin-ui/package.json b/cmd/relay/relay-admin-ui/package.json new file mode 100644 index 000000000..4cd422467 --- /dev/null +++ b/cmd/relay/relay-admin-ui/package.json @@ -0,0 +1,35 @@ +{ + "name": "relay-admin-ui", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "preview": "vite preview" + }, + "dependencies": { + "@headlessui/react": "^1.7.15", + "@heroicons/react": "^2.0.18", + "@tailwindcss/forms": "^0.5.4", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "react-router-dom": "^6.14.2" + }, + "devDependencies": { + "@types/react": "^18.2.14", + "@types/react-dom": "^18.2.6", + "@typescript-eslint/eslint-plugin": "^5.61.0", + "@typescript-eslint/parser": "^5.61.0", + "@vitejs/plugin-react": "^4.0.1", + "autoprefixer": "^10.4.14", + "eslint": "^8.44.0", + "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-refresh": "^0.4.1", + "postcss": "^8.4.26", + "tailwindcss": "^3.3.3", + "typescript": "^5.0.2", + "vite": "^4.4.0" + } +} diff --git a/cmd/relay/relay-admin-ui/postcss.config.js b/cmd/relay/relay-admin-ui/postcss.config.js new file mode 100644 index 000000000..2e7af2b7f --- /dev/null +++ b/cmd/relay/relay-admin-ui/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/cmd/relay/relay-admin-ui/public/vite.svg b/cmd/relay/relay-admin-ui/public/vite.svg new file mode 100644 index 000000000..e7b8dfb1b --- /dev/null +++ b/cmd/relay/relay-admin-ui/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/cmd/relay/relay-admin-ui/src/App.css b/cmd/relay/relay-admin-ui/src/App.css new file mode 100644 index 000000000..5496fba67 --- /dev/null +++ b/cmd/relay/relay-admin-ui/src/App.css @@ -0,0 +1,25 @@ +.fade-in { + animation: fadeIn 0.5s; +} + +.fade-out { + animation: fadeOut 0.5s; +} + +@keyframes fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } +} + +@keyframes fadeOut { + from { + opacity: 1; + } + to { + opacity: 0; + } +} diff --git a/cmd/relay/relay-admin-ui/src/App.tsx b/cmd/relay/relay-admin-ui/src/App.tsx new file mode 100644 index 000000000..88b21396c --- /dev/null +++ b/cmd/relay/relay-admin-ui/src/App.tsx @@ -0,0 +1,271 @@ +import "./App.css"; +import { + NavLink, + RouterProvider, + createBrowserRouter, + useNavigate, +} from "react-router-dom"; +import Dash from "./components/Dash/Dash"; +import { Disclosure } from "@headlessui/react"; +import { Bars3Icon, XMarkIcon } from "@heroicons/react/24/outline"; +import Login from "./components/Login/Login"; +import { useEffect } from "react"; +import Logout from "./components/Logout/Logout"; +import Domains from "./components/Domains/Domains"; +import Repos from "./components/Repos/Repos"; +import Consumers from "./components/Consumers/Consumers"; +import NewPDS from "./components/NewPDS/NewPDS"; + +function classNames(...classes: string[]) { + return classes.filter(Boolean).join(" "); +} + +// Redirect to /login if not authenticated +function RequireAuth({ children }: { children: React.ReactNode }) { + const navigate = useNavigate(); + + useEffect(() => { + if (!localStorage.getItem("admin_route_token")) { + navigate("/login"); + } + }, []); + + return children; +} + +interface Route { + path: string; + name: string; + element: React.ReactNode; + requrieAuth?: boolean; + hideIfAuth?: boolean; +} + +const routes: Route[] = [ + { + path: "/", + name: "PDS List", + element: ( + +