Merge branch 'opensearch-project:main' into main

Author: lucy66hw

.bazelrc

@@ -8,3 +8,6 @@ build --host_cxxopt=-std=c++14
 
 # Enable deprecated java_common.compile parameters
 build --incompatible_java_common_parameters=false
+
+# Disabel bazelmod by default
+build --noenable_bzlmod

.bazelversion

@@ -1 +1 @@
-5.4.1
+7.0.0

.github/workflows/build-protobufs-go.yml

@@ -0,0 +1,18 @@
+name: Build OpenSearch Protobufs GO Artifacts
+
+on:
+  pull_request:
+
+jobs:
+  build-protobufs-go:
+    runs-on: ubuntu-latest
+    if: github.repository == 'opensearch-project/opensearch-protobufs'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 21
+      - name: build protobufs go
+        run: bazel build //:go_protos_all

.github/workflows/build-protobufs-python.yml

@@ -0,0 +1,18 @@
+name: Build OpenSearch Protobufs PYTHON Artifacts
+
+on:
+  pull_request:
+
+jobs:
+  build-protobufs-python:
+    runs-on: ubuntu-latest
+    if: github.repository == 'opensearch-project/opensearch-protobufs'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 21
+      - name: build protobufs python
+        run: bazel build //:python_protos_all

.github/workflows/maven-publish.yml

@@ -19,15 +19,18 @@ jobs:
         with:
           distribution: temurin # Temurin is a distribution of adoptium
           java-version: 21
-      - uses: aws-actions/configure-aws-credentials@v1
+
+      - name: Load secret
+        uses: 1password/load-secrets-action@v2
         with:
-          role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
-          aws-region: us-east-1
+          # Export loaded secrets as environment variables
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
+          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
+
       - name: publish snapshots to maven
         run: |
-          export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
-          export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
-          echo "::add-mask::$SONATYPE_USERNAME"
-          echo "::add-mask::$SONATYPE_PASSWORD"
           ./tools/java/package_proto_jar.sh -c true
           ./gradlew publishCreatePublicationToSnapshotsRepository

BUILD.bazel

@@ -1,30 +1,39 @@
-# protos/BUILD.bazel
-load("@rules_java//java:defs.bzl", "java_proto_library")
-load("@io_grpc_grpc_java//:java_grpc_library.bzl", "java_grpc_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 package(default_visibility = ["//visibility:public"])
 
-# Java proto library
-java_proto_library(
-    name = "protos_java",
-    deps = [
-        "//protos/services:document_service_proto",
-        "//protos/services:search_service_proto"
-    ],
+java_library(
+    name = "java_protos_all",
     visibility = ["//visibility:public"],
+    exports = [
+        "//protos/schemas:common_java_proto",
+        "//protos/schemas:document_java_proto",
+        "//protos/schemas:search_java_proto",
+        "//protos/services:document_service_java_proto",
+        "//protos/services:search_service_java_proto",
+    ],
 )
 
-# Java gRPC library
-java_grpc_library(
-    name = "document_service_grpc_java",
-    srcs = ["//protos/services:document_service_proto"],
-    deps = [":protos_java"],
+py_library(
+    name = "python_protos_all",
     visibility = ["//visibility:public"],
+    deps = [
+        "//protos/schemas:common_python_proto",
+        "//protos/schemas:document_python_proto",
+        "//protos/schemas:search_python_proto",
+        "//protos/services:document_service_python_proto",
+        "//protos/services:search_service_python_proto",
+    ],
 )
 
-java_grpc_library(
-    name = "search_service_grpc_java",
-    srcs = ["//protos/services:search_service_proto"],
-    deps = [":protos_java"],
+go_library(
+    name = "go_protos_all",
     visibility = ["//visibility:public"],
+    deps = [
+        "//protos/schemas:common_go_proto",
+        "//protos/schemas:document_go_proto",
+        "//protos/schemas:search_go_proto",
+        "//protos/services:document_service_go_proto",
+        "//protos/services:search_service_go_proto",
+    ],
 )

CHANGELOG.md

@@ -4,16 +4,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased]
 ### Added
-- Add search API to generated proto ([#82](https://github.com/opensearch-project/opensearch-protobufs/pull/82))
-- Replace required property name from _* to underscore_* and remove duplicate __ from modified title ([#88](https://github.com/opensearch-project/opensearch-protobufs/pull/88))
-- Preprocessing spec for collapse oneOf if childComplexOneOf property contains child SimpleOneOf. ([#89](https://github.com/opensearch-project/opensearch-protobufs/pull/89))
-- Preprocessing spec for reconstructing a standard schema from single-key map. ([#90](https://github.com/opensearch-project/opensearch-protobufs/pull/90))
-- Using official protobuf generator. ([#91](https://github.com/opensearch-project/opensearch-protobufs/pull/91))
-- Preprocessing spec for deduplicating enum values. ([#93](https://github.com/opensearch-project/opensearch-protobufs/pull/93))
-- Address Generator Build Failure. ([#95](https://github.com/opensearch-project/opensearch-protobufs/pull/95))
-- Add http code at the end of response. ([#97](https://github.com/opensearch-project/opensearch-protobufs/pull/97))
-- Add GRPC Search server-side streaming endpoint ([#98](https://github.com/opensearch-project/opensearch-protobufs/pull/98))
-- Add bi-directional streaming Bulk GRPC endpoint ([#101](https://github.com/opensearch-project/opensearch-protobufs/pull/101))
+- Add Go protobuf generation support ([#131](https://github.com/opensearch-project/opensearch-protobufs/pull/131))
 
 ### Removed
 

DEVELOPER_GUIDE.md

@@ -2,9 +2,27 @@
 Install bazel, using the version in .bazelversion.
 
 # Compile protos and grpc
+
+## All artifacts
 ```
 bazel build //...
 ```
+
+## Java
+```
+bazel build //:java_protos_all
+```
+
+## Python
+```
+bazel build //:python_protos_all
+```
+
+## Go
+```
+bazel build //:go_protos_all
+```
+
 # Proto generated code
 ## Java
 ### Generate Java Code and packaging as a Maven/Gradle dependency
@@ -14,7 +32,7 @@ To package the generated Java files into a Maven-compatible JAR that can be used
 #optional
 rm -rf bazel*
 
-rm -rf generated && bazel build //... && ./tools/java/package_proto_jar.sh
+rm -rf generated && bazel build //:java_protos_all && ./tools/java/package_proto_jar.sh
 ```
 
 This script will:
@@ -47,7 +65,7 @@ If using snapshot jar uploaded to sonatype:
 ```
 repositories {
   maven {
-    url = 'https://aws.oss.sonatype.org/content/repositories/snapshots'
+    url = 'https://central.sonatype.com/repository/maven-snapshots/'
   }
 }
 

Dockerfile

@@ -0,0 +1,128 @@
+FROM ubuntu:22.04 AS base-bazel
+
+ENV BAZEL_VERSION=7.0.0
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    git \
+    zip \
+    unzip \
+    openjdk-21-jdk \
+    maven \
+    g++ \
+    gcc \
+    golang-go \
+    && apt-get clean
+
+RUN apt-get install software-properties-common -y
+RUN add-apt-repository ppa:deadsnakes/ppa -y
+RUN apt-get install python3.10 python3.10-dev -y
+
+RUN curl -fsSL https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh -o bazel-installer.sh \
+    && chmod +x bazel-installer.sh \
+    && ./bazel-installer.sh \
+    && rm bazel-installer.sh
+
+RUN bazel --version
+
+FROM base-bazel AS build-bazel
+
+# Run as non-root - Required for rules_python
+# See: https://github.com/bazelbuild/rules_python/pull/713
+# Create group and user
+RUN groupadd -r bazeluser && useradd -r -m -g bazeluser bazeluser
+RUN mkdir -p /build && \
+    chown -R bazeluser:bazeluser /build && \
+    chmod -R 777 /build
+
+USER bazeluser
+WORKDIR /build
+
+# Copy entire repository for convenience
+# Invalidate cache to ensure updates are captured
+ARG CACHEBUST=1
+COPY --chown=bazeluser:bazeluser . .
+
+#################################################
+##### JAVA STAGES ###############################
+#################################################
+
+FROM build-bazel AS build-bazel-java
+
+RUN bazel build //:java_protos_all
+
+FROM build-bazel-java AS package-bazel-java
+
+RUN /build/tools/java/package_proto_jar.sh
+
+FROM package-bazel-java AS test-bazel-java
+
+ARG OPENSEARCH_BRANCH=main
+ARG PROTO_SNAPSHOT_VERSION=0.4.0
+
+ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64
+ENV PATH=$PATH:$JAVA_HOME/bin
+ENV PUB_ARTIFACTS=/build/generated/maven/publish/
+
+RUN mvn install:install-file \
+  -Dfile=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT.jar \
+  -DpomFile=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT.pom \
+  -Dsources=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT-sources.jar \
+  -Djavadoc=${PUB_ARTIFACTS}protobufs-${PROTO_SNAPSHOT_VERSION}-SNAPSHOT-javadoc.jar
+
+RUN git clone --branch ${OPENSEARCH_BRANCH} https://github.com/opensearch-project/OpenSearch.git /build/opensearch
+
+WORKDIR /build/opensearch
+
+# Update transport-grpc/build.gradle to use PROTO_SNAPSHOT_VERSION
+RUN sed -i 's/org\.opensearch:protobufs:[0-9]\+\.[0-9]\+\.[0-9]\+/org.opensearch:protobufs:'"${PROTO_SNAPSHOT_VERSION}"'/' /build/opensearch/plugins/transport-grpc/build.gradle
+
+RUN ./gradlew :plugins:transport-grpc:test -Drepos.mavenLocal
+RUN ./gradlew :plugins:transport-grpc:internalClusterTest -Drepos.mavenLocal
+
+#################################################
+##### PYTHON STAGES #############################
+#################################################
+
+FROM build-bazel AS build-bazel-python
+
+RUN bazel build //:python_protos_all
+
+#################################################
+##### GO STAGES ##################################
+#################################################
+
+FROM build-bazel AS build-bazel-go
+
+RUN bazel build //:go_protos_all
+
+FROM build-bazel-go AS package-bazel-go
+
+# Create a clean directory structure for Go protobuf files
+RUN mkdir -p /build/generated/go/opensearchpb && \
+    mkdir -p /build/generated/go/services
+
+# Copy generated Go protobuf files
+RUN find bazel-bin/protos/schemas -name "*.pb.go" -path "*_go_proto_pb*" -exec cp {} /build/generated/go/opensearchpb/ \; && \
+    find bazel-bin/protos/services -name "*.pb.go" -path "*_go_proto_pb*" -exec cp {} /build/generated/go/services/ \; && \
+    find bazel-bin/protos/services -name "*_grpc.pb.go" -path "*_go_proto_pb*" -exec cp {} /build/generated/go/services/ \;
+
+FROM package-bazel-go AS test-bazel-go
+
+# Set up Go module and validate generated code
+WORKDIR /build/generated/go
+
+# Create go.mod with proper module path
+RUN echo 'module github.com/opensearch-project/opensearch-protobufs/go' > go.mod && \
+    echo '' >> go.mod && \
+    echo 'go 1.19' >> go.mod && \
+    echo '' >> go.mod && \
+    echo 'require (' >> go.mod && \
+    echo '    google.golang.org/protobuf v1.31.0' >> go.mod && \
+    echo '    google.golang.org/grpc v1.58.0' >> go.mod && \
+    echo ')' >> go.mod
+
+# Test that Go code compiles successfully
+RUN go mod tidy && \
+    go build ./...