opensearch-hadoop/.github/workflows/publish_snapshot.yml at 696ef647293d5f2583bd217d40716787039f5848 · gaiksaya/opensearch-hadoop · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
name: Publish snapshots to maven

on:
  push:
    branches:
      - main

jobs:
  build-and-publish-snapshots:
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: write
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

      - name: Set up JDK 11
        uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3
        with:
          distribution: 'temurin'
          java-version: '11'
          cache: 'gradle'

      - name: 'Setup: Java 11 env'
        run: echo "JAVA11_HOME=$JAVA_HOME" >> $GITHUB_ENV

      - name: Set up JDK 17
        uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3
        with:
          distribution: 'temurin'
          java-version: '17'
          cache: 'gradle'

      - name: 'Setup: Java 17 env'
        run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV

      - name: Set up JDK 21
        uses: actions/setup-java@17f84c3641ba7b8f6deff6309fc4c864478f5d62 # v3
        with:
          distribution: 'temurin'
          java-version: '21'
          cache: 'gradle'

      - name: 'Setup: Java 21 env'
        run: echo "JAVA21_HOME=$JAVA_HOME" >> $GITHUB_ENV

      - name: Read value from Properties-file
        id: read_property
        uses: christian-draeger/read-properties@908f99d3334be3802ec7cb528395a69d19914e7b # 1.1.1
        with:
          path: 'buildSrc/opensearch-hadoop-version.properties'
          properties: 'opensearch_hadoop'

      - name: set required env variables
        run: |
          echo "MAVEN_HOME=maven" >> $GITHUB_ENV
          echo "MAVEN_HOME_CLIENT=maven/org/opensearch/client" >> $GITHUB_ENV
          echo "VERSION=${{ steps.read_property.outputs.opensearch_hadoop }}" >> $GITHUB_ENV

      - name: make snapshot directories
        run: |
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}"
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop/${{ env.VERSION }}-SNAPSHOT"
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop-mr/${{ env.VERSION }}-SNAPSHOT"
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop-hive/${{ env.VERSION }}-SNAPSHOT"
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-30_2.12/${{ env.VERSION }}-SNAPSHOT"
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-30_2.13/${{ env.VERSION }}-SNAPSHOT"
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-35_2.12/${{ env.VERSION }}-SNAPSHOT"
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-35_2.13/${{ env.VERSION }}-SNAPSHOT"
          mkdir -p "${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-40_2.13/${{ env.VERSION }}-SNAPSHOT"

      - name: Setup Gradle
        uses: gradle/gradle-build-action@a8f75513eafdebd8141bd1cd4e30fcd194af8dfa # v2

      - name: generate JARs
        run: |
          ./gradlew clean -S -Dbuild.snapshot=true -Dorg.gradle.warning.mode=summary distribution --no-configuration-cache --info

      - name: generate sha and md5
        run: |
          for i in `find . -path '*/distributions/*' -name "*.jar" -type f`; do sha1sum "$i" >> "$i.sha1"; done
          for i in `find . -path '*/poms/*' -name "*.pom" -type f`; do sha1sum "$i" >> "$i.sha1"; done
          for i in `find . -path '*/distributions/*' -name "*.jar" -type f`; do sha512sum "$i" >> "$i.sha512"; done
          for i in `find . -path '*/poms/*' -name "*.pom" -type f`; do sha512sum "$i" >> "$i.sha512"; done
          for i in `find . -path '*/distributions/*' -name "*.jar" -type f`; do sha256sum "$i" >> "$i.sha256"; done
          for i in `find . -path '*/poms/*' -name "*.pom" -type f`; do sha256sum "$i" >> "$i.sha256"; done
          for i in `find . -path '*/distributions/*' -name "*.jar" -type f`; do md5sum "$i" >> "$i.md5"; done
          for i in `find . -path '*/poms/*' -name "*.pom" -type f`; do md5sum "$i" >> "$i.md5"; done

      - name: copy JARs and POMs
        run: |
          rm -rf dist/build/distributions/*.zip
          cp -R mr/build/distributions/* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop-mr/${{ env.VERSION }}-SNAPSHOT
          cp -R mr/build/poms/* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop-mr/${{ env.VERSION }}-SNAPSHOT
          cp -R hive/build/distributions/* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop-hive/${{ env.VERSION }}-SNAPSHOT
          cp -R hive/build/poms/* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop-hive/${{ env.VERSION }}-SNAPSHOT
          cp -R dist/build/distributions/* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop/${{ env.VERSION }}-SNAPSHOT
          cp -R dist/build/poms/* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-hadoop/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-30/build/distributions/opensearch-spark-30_2.12* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-30_2.12/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-30/build/poms/opensearch-spark-30_2.12* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-30_2.12/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-30/build/distributions/opensearch-spark-30_2.13* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-30_2.13/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-30/build/poms/opensearch-spark-30_2.13* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-30_2.13/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-35/build/distributions/opensearch-spark-35_2.12* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-35_2.12/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-35/build/poms/opensearch-spark-35_2.12* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-35_2.12/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-35/build/distributions/opensearch-spark-35_2.13* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-35_2.13/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-35/build/poms/opensearch-spark-35_2.13* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-35_2.13/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-40/build/distributions/opensearch-spark-40_2.13* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-40_2.13/${{ env.VERSION }}-SNAPSHOT
          cp -R spark/sql-40/build/poms/opensearch-spark-40_2.13* ${{ env.MAVEN_HOME_CLIENT }}/opensearch-spark-40_2.13/${{ env.VERSION }}-SNAPSHOT

      - name: debug output directories
        run: tree maven

      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          repository: 'opensearch-project/opensearch-build'
          path: 'build'
          ref: 'main'

      - name: Load secret
        uses: 1password/load-secrets-action@581a835fb51b8e7ec56b71cf2ffddd7e68bb25e0 # v2
        with:
          # Export loaded secrets as environment variables
          export-env: true
        env:
          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
          MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
          MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role

      - name: Export SNAPSHOT_REPO_URL
        run: |
          snapshot_repo_url=${{ env.MAVEN_SNAPSHOTS_S3_REPO }}
          echo "SNAPSHOT_REPO_URL=$snapshot_repo_url" >> $GITHUB_ENV

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5
        with:
          role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
          aws-region: us-east-1

      - name: Get credentials and publish snapshots to maven
        run: |
          build/publish/publish-snapshot.sh $MAVEN_HOME