Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# config for Pinterest ktlint. https://github.com/pinterest/ktlint

root = true

[*.{kt,kts}]
ktlint_code_style = intellij_idea
# we have detekt also checking for max line length. Disable the linter and use only one tool to check for max line length.
# See https://github.com/arturbosch/detekt
ktlint_standard_max-line-length = disabled
ktlint_ignore_back_ticked_identifier = true

ktlint_standard_function-naming = disabled
ktlint_standard_property-naming = disabled
ktlint_standard_function-signature = disabled
ktlint_standard_value-argument-comment = disabled
ktlint_standard_argument-list-wrapping = disabled
ktlint_standard_value-parameter-comment = disabled
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ contact_links:
about: Please ask and answer questions here.
- name: AWS/Amazon Security
url: https://aws.amazon.com/security/vulnerability-reporting/
about: Please report security vulnerabilities here.
about: Please report security vulnerabilities here.
18 changes: 18 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
version: 2
updates:
- package-ecosystem: "gradle"
directory: "/"
schedule:
interval: "weekly"
commit-message:
prefix: "dependabot:"
ignore:
# For all packages, ignore all major versions to minimize breaking issues
- dependency-name: "*"
update-types: ["version-update:semver-major"]
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
commit-message:
prefix: "dependabot:"
38 changes: 19 additions & 19 deletions .github/workflows/add-untriaged.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
name: Apply 'untriaged' label during issue lifecycle
on:
issues:
types: [opened, reopened, transferred]
jobs:
apply-label:
runs-on: ubuntu-latest
steps:
- uses: actions/github-script@v6
with:
script: |
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ['untriaged']
})
name: Apply 'untriaged' label during issue lifecycle

on:
issues:
types: [opened, reopened, transferred]

jobs:
apply-label:
runs-on: ubuntu-latest
steps:
- uses: actions/github-script@v6
with:
script: |
github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: ['untriaged']
})
2 changes: 1 addition & 1 deletion .github/workflows/auto-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ jobs:
- uses: ncipollo/release-action@v1
with:
github_token: ${{ steps.github_app_token.outputs.token }}
bodyFile: release-notes/opensearch-cross-cluster-replication.release-notes-${{steps.tag.outputs.tag}}.md
bodyFile: release-notes/opensearch-cross-cluster-replication.release-notes-${{steps.tag.outputs.tag}}.md
6 changes: 3 additions & 3 deletions .github/workflows/delete_backport_branch.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
name: Delete merged branch of the backport PRs
on:
on:
pull_request:
types:
- closed

jobs:
delete-branch:
runs-on: ubuntu-latest
Expand All @@ -12,4 +12,4 @@ jobs:
- name: Delete merged branch
uses: SvanBoxel/delete-merged-branch@main
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
64 changes: 32 additions & 32 deletions .github/workflows/maven-publish.yml
Original file line number Diff line number Diff line change
@@ -1,32 +1,32 @@
name: Publish snapshots to maven
on:
workflow_dispatch:
push:
branches:
- main
- '[0-9]+.[0-9]+'
- '[0-9]+.x'
jobs:
build-and-publish-snapshots:
runs-on: ubuntu-latest
permissions:
id-token: write
contents: write
steps:
- uses: actions/setup-java@v4
with:
distribution: temurin # Temurin is a distribution of adoptium
java-version: 21
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
aws-region: us-east-1
- name: publish snapshots to maven
run: |
export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
echo "::add-mask::$SONATYPE_USERNAME"
echo "::add-mask::$SONATYPE_PASSWORD"
./gradlew publishPluginZipPublicationToSnapshotsRepository
name: Publish snapshots to maven

on:
workflow_dispatch:
push:
branches:
- main
- '[0-9]+.[0-9]+'
- '[0-9]+.x'
jobs:
build-and-publish-snapshots:
runs-on: ubuntu-latest
permissions:
id-token: write
contents: write
steps:
- uses: actions/setup-java@v4
with:
distribution: temurin # Temurin is a distribution of adoptium
java-version: 21
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
aws-region: us-east-1
- name: publish snapshots to maven
run: |
export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
echo "::add-mask::$SONATYPE_USERNAME"
echo "::add-mask::$SONATYPE_PASSWORD"
./gradlew publishPluginZipPublicationToSnapshotsRepository
6 changes: 3 additions & 3 deletions .github/workflows/security-knn-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ jobs:
name: "plugin check"
run: |
opensearch_version=$(./gradlew properties | grep -E '^version:' | awk '{print $2}')
# we publish build artifacts to the below url
# we publish build artifacts to the below url
sec_plugin_url="https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/opensearch-security/"$opensearch_version"/"
sec_st=$(curl -s -o /dev/null -w "%{http_code}" $sec_plugin_url)
if [ "$sec_st" = "200" ]; then
Expand All @@ -53,7 +53,7 @@ jobs:
else
echo "isKnnPluginAvailable=False" >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
fi
fi

build-linux:
needs: [req, Get-CI-Image-Tag]
Expand Down Expand Up @@ -107,7 +107,7 @@ jobs:
uses: codecov/[email protected]

knn-build-linux:
needs: [req, Get-CI-Image-Tag]
needs: [req, Get-CI-Image-Tag]
if: ${{ 'True' == needs.req.outputs.isKnnPluginAvailable }}
# Job name
name: Build and Run Knn tests
Expand Down
10 changes: 5 additions & 5 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ As with other types of contributions, the first step is to [open an issue on Git

## Becoming a maintainer
You can become a maintainer by actively contributing to any project, and being nominated by an existing maintainer.
Refer to [this](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#becoming-a-maintainer) for the nomination process.
Refer to [this](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#becoming-a-maintainer) for the nomination process.

## Developer Certificate of Origin

Expand Down Expand Up @@ -104,7 +104,7 @@ We run BWC test suite on all the changes to ensure that we do not break backward
Run this command to trigger the complete bwc testsuite`./gradlew bwcTestSuite`

Test suite covers following 3 scenarios:
1. Mixed cluster: We create 3 node leader and follower cluster and upgrade one node on both clusters. After the upgrade, we verify that the ongoing replication keeps working as expected. Run this individual test using ``./gradlew mixedClusterTask``
1. Mixed cluster: We create 3 node leader and follower cluster and upgrade one node on both clusters. After the upgrade, we verify that the ongoing replication keeps working as expected. Run this individual test using ``./gradlew mixedClusterTask``
2. Rolling upgrade: We create 3 node leader and follower cluster and upgrade all nodes one by one. After the upgrade, we verify that the ongoing replication keeps working as expected. Run this individual test using ``./gradlew rollingUpgradeClusterTask``
3. Full cluster restart: We create 3 node leader and follower cluster and upgrade all nodes simultaneously. After the upgrade, we verify that the ongoing replication keeps working as expected. Run this individual test using ``./gradlew fullRestartClusterTask``

Expand Down Expand Up @@ -147,18 +147,18 @@ merged to main, the workflow will create a backport PR to the `1.x` branch.

## Security tests

To run security tests locally,
To run security tests locally,
```
git clone https://github.com/opensearch-project/security.git
cd security
cd security
./gradlew clean build -Dbuild.snapshot=false -x test
# REPLICATION_DIR - root dir of cross-cluster-replication repo
cp build/distributions/opensearch-security-1.3.0.0.zip $REPLICATION_DIR/src/test/resources/security/plugin/opensearch-security.zip
./gradlew clean release -Psecurity=true
```

## Performance tests

To run Performance tests locally,
```
bin/run_perf_test.sh --bundle-manifest path_to_OS_bundle_manifest --config config_path --without-security
Expand Down
9 changes: 4 additions & 5 deletions HANDBOOK.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ Content-Type: application/json

```

**Example**
**Example**
```bash
curl -k -u testuser:testuser -XPUT \
"https://${FOLLOWER}/_plugins/_replication/follower-01/_start?pretty" \
Expand Down Expand Up @@ -214,7 +214,7 @@ curl -k -u testuser:testuser -XPOST \
-H 'Content-type: application/json' -d'{}'

# You can confirm data isn't replicated any more by making modifications to
# leader-01 index on $LEADER cluster
# leader-01 index on $LEADER cluster
```

## Start replication via Autofollow pattern
Expand Down Expand Up @@ -275,7 +275,7 @@ curl -k -u testuser:testuser -XDELETE \
Stats for all AutoFollow tasks running on the follower cluster can be checked using `autofollow_stats`.

```bash
curl -k -u admin:admin -XGET "https://${FOLLOWER}/_plugins/_replication/autofollow_stats"
curl -k -u admin:admin -XGET "https://${FOLLOWER}/_plugins/_replication/autofollow_stats"
{
"num_success_start_replication" : 16,
"num_failed_start_replication" : 0,
Expand Down Expand Up @@ -304,7 +304,7 @@ curl -k -u admin:admin -XGET "https://${FOLLOWER}/_plugins/_replication/autofoll
Until a status API is added, you can check ongoing replication via the tasks API.

```bash
curl -k -u admin:admin -XGET "https://${FOLLOWER}/_cat/tasks?v&actions=*replication*&detailed"
curl -k -u admin:admin -XGET "https://${FOLLOWER}/_cat/tasks?v&actions=*replication*&detailed"

action task_id parent_task_id type start_time timestamp running_time ip node description
cluster:indices/admin/replication[c] ltIs84uTRLOYnOr8Giu0VQ:118 cluster:1 persistent 1613651479438 12:31:19 17.7s 172.18.0.20 odfe-follower1 replication:leader-cluster:[leader-01/g3d9ddwZQHeuvEGEouQxDQ] -> follower-01
Expand Down Expand Up @@ -395,4 +395,3 @@ curl -k -u admin:admin -XGET "https://${FOLLOWER}/.tasks/_search?pretty"
}
}
```

10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
- [License](#license)


Cross-Cluster Replication Plugin enables users to replicate data across two opensearch clusters which enables a number of use cases such as
Cross-Cluster Replication Plugin enables users to replicate data across two opensearch clusters which enables a number of use cases such as

- **Disaster Recovery (DR) or High Availability (HA):** For production systems with high availability requirements, cross-cluster replication provides the safety-net of being able to failover to an alternate cluster in case of failure or outages on the primary cluster.
- **Reduced Query Latency:** For critical business needs, responding to the customer query in the shortest time is critical. Replicating data to a cluster that is closest to the user can drastically reduce the query latency. Applications can redirect the customer query to the nearest data center where data has been replicated.
Expand All @@ -32,9 +32,9 @@ Following are the tenets that guided our design:

- **Secure**: Cross-cluster replication should offer strong security controls for all flows and APIs.
- **Correctness**: There must be no difference between the intended contents of the follower index and the leader index.
- **Performance**: Replication should not impact indexing rate of the leader cluster.
- **Performance**: Replication should not impact indexing rate of the leader cluster.
- **Lag**: The replication lag between the leader and the follower cluster should be under a few seconds.
- **Resource usage**: Replication should use minimal resources.
- **Resource usage**: Replication should use minimal resources.


The replication machinery is implemented as an OpenSearch plugin that exposes APIs to control replication, spawns background persistent tasks to asynchronously replicate indices and utilizes snapshot repository abstraction to facilitate bootstrap. Replication relies on cross cluster connection setup from the follower cluster to the leader cluster for connectivity. Once replication is initiated on an index, a background persistent task per primary shard on the follower cluster continuously polls corresponding shards from the leader index and applies the changes on to the follower shard. The replication plugin offers seamless integration with the OpenSearch Security plugin for secure data transfer and access control.
Expand All @@ -45,7 +45,7 @@ The replication machinery is implemented as an OpenSearch plugin that exposes AP
The project in this package uses the [Gradle](https://docs.gradle.org/current/userguide/userguide.html) build system. Gradle comes with excellent documentation that should be your first stop when trying to figure out how to operate or modify the build.

### Building from the command line
Set JAVA_HOME to JDK-21 or above
Set JAVA_HOME to JDK-21 or above

1. `./gradlew build` builds and tests project.
2. `./gradlew clean release` cleans previous builds, creates new build and tests project.
Expand Down Expand Up @@ -146,7 +146,7 @@ curl -XPOST "http://${FOLLOWER}/_plugins/_replication/follower-01/_stop?pretty"
-H 'Content-type: application/json' -d'{}'

# You can confirm data isn't replicated any more by making modifications to
# leader-01 index on $LEADER cluster
# leader-01 index on $LEADER cluster
```

For much detailed instructions/examples please refer to [HANDBOOK](HANDBOOK.md).
Expand Down
9 changes: 7 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ plugins {
id 'com.netflix.nebula.ospackage' version "11.6.0"
id "com.dorongold.task-tree" version "1.5"
id "jacoco"
id "com.diffplug.spotless" version "6.25.0"
}

allprojects {
Expand All @@ -111,6 +112,7 @@ apply plugin: 'org.jetbrains.kotlin.jvm'
apply plugin: 'org.jetbrains.kotlin.plugin.allopen'
apply plugin: 'opensearch.pluginzip'
apply plugin: 'opensearch.java-agent'
apply from: 'gradle/formatting.gradle'

forbiddenApisTest.ignoreFailures = true

Expand Down Expand Up @@ -141,10 +143,13 @@ configurations {
opensearchPlugin
}

configurations.all {
exclude group:"org.jetbrains.kotlin", module: "kotlin-stdlib-jdk8"
exclude group:"org.jetbrains.kotlin", module: "kotlin-stdlib-jdk7"
}

dependencies {
runtimeOnly "org.opensearch:opensearch:${opensearch_version}"
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7"
implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
implementation "org.jetbrains:annotations:13.0"
Expand Down
8 changes: 8 additions & 0 deletions formatter/license-header.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/
Loading
Loading