diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77b4979a7f55..8ea3f9cb01e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -433,7 +433,7 @@ jobs: with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} @@ -448,7 +448,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" source: "onedir" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 484d0c66b0f1..b9132860779f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -486,7 +486,7 @@ jobs: with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} @@ -501,7 +501,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" source: "onedir" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} @@ -521,7 +521,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" source: "src" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 3fe1f8986e41..b5b3169e48bd 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -476,7 +476,7 @@ jobs: with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} @@ -491,7 +491,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" source: "onedir" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 64a53a26e2f4..d5a9d22fe6ab 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -473,7 +473,7 @@ jobs: with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} linux_arm_runner: ${{ fromJSON(needs.prepare-workflow.outputs.config)['linux_arm_runner'] }} @@ -489,7 +489,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" source: "onedir" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} @@ -510,7 +510,7 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - relenv-version: "0.19.0" + relenv-version: "0.19.2" python-version: "3.10.17" source: "src" matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }} diff --git a/changelog/66627.added.md b/changelog/66627.added.md new file mode 100644 index 000000000000..a07df6b4f2d7 --- /dev/null +++ b/changelog/66627.added.md @@ -0,0 +1 @@ +Added the ability to configure the cluster event port and added documentation for it diff --git a/cicd/shared-gh-workflows-context.yml b/cicd/shared-gh-workflows-context.yml index 95b0830d90b0..df317a3d8388 100644 --- a/cicd/shared-gh-workflows-context.yml +++ b/cicd/shared-gh-workflows-context.yml @@ -1,6 +1,6 @@ nox_version: "2022.8.7" python_version: "3.10.17" -relenv_version: "0.19.0" +relenv_version: "0.19.2" release_branches: - "3006.x" - "3007.x" diff --git a/conf/master b/conf/master index 1e951bacfded..ab37e280ef2b 100644 --- a/conf/master +++ b/conf/master @@ -1357,3 +1357,29 @@ # Set a list of clients to enable in in the API #netapi_enable_clients: [] + + +##### Master Cluster Settings ##### +##################################################### +# Settings for configuring salt master to work in a cluster configuration behind +# a load balancer. + +# When defined, the master will operate in cluster mode. The master will send +# the cluster key and id to minions instead of its own key and id. The master +# will also forward its local event bus to other masters defined by +# ``cluster_peers`` +#cluster_id: master_cluster + +# Defines the other masters in the cluster. Can be IP addresses or hostnames +#cluster_peers: +# - master 2 +# - master 3 + +# When ``cluster_pki_dir`` is defined, this sets the location of where this +# cluster will store its cluster public and private key as well as any minion +# keys. This setting will default to the value of ``pki_dir``, but should be +# changed to the filesystem location shared between peers in the cluster. +#cluster_pki_dir: /my/gluster/share/pki + +# The port required to be open for a master cluster to properly function +#cluster_pool_port: 4520 diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 3db59a1aa1a1..5d08d4281b34 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -218,10 +218,9 @@ When defined, the master will operate in cluster mode. The master will send the cluster key and id to minions instead of its own key and id. The master will also forward its local event bus to other masters defined by ``cluster_peers`` - .. code-block:: yaml - cluster_id: master + cluster_id: master_cluster .. conf_master:: cluster_peers @@ -230,8 +229,8 @@ also forward its local event bus to other masters defined by ``cluster_peers`` .. versionadded:: 3007 -When ``cluster_id`` is defined, this setting is a list of other master -(hostnames or ips) that will be in the cluster. +When ``cluster_peers`` is defined, this setting is a list of other master +(hostnames or IPs) that will be in the cluster. .. code-block:: yaml @@ -246,15 +245,29 @@ When ``cluster_id`` is defined, this setting is a list of other master .. versionadded:: 3007 -When ``cluster_id`` is defined, this sets the location of where this cluster -will store its cluster public and private key as well as any minion keys. This -setting will default to the value of ``pki_dir``, but should be changed -to the filesystem location shared between peers in the cluster. +When ``cluster_pki_dir`` is defined, this sets the location of where this +cluster will store its cluster public and private key as well as any minion +keys. This setting will default to the value of ``pki_dir``, but should be +changed to the filesystem location shared between peers in the cluster. .. code-block:: yaml - cluster_pki: /my/gluster/share/pki + cluster_pki_dir: /my/gluster/share/pki + + +.. conf_master:: cluster_port + +``cluster_pool_port`` +--------------------- + +.. versionadded:: 3007.2 + +When ``cluster_pool_port`` is defined, it sets the TCP port number HAProxy +listens on for incoming TCP connections. The default is ``4520`` + +.. code-block:: yaml + cluster_pool_port: 4520 .. conf_master:: extension_modules diff --git a/doc/topics/tutorials/master-cluster.rst b/doc/topics/tutorials/master-cluster.rst index 7b7ae25f11dd..10910643ad9a 100644 --- a/doc/topics/tutorials/master-cluster.rst +++ b/doc/topics/tutorials/master-cluster.rst @@ -32,7 +32,7 @@ when operating as a cluster, minion keys are stored in the `cluster_pki_dir` instead of the master's `pki_dir`. -Reference Implimentation +Reference Implementation ======================== Gluster: https://docs.gluster.org/en/main/Quick-Start-Guide/Quickstart/ diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 80266728c5a1..cf41575fcd5b 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -1,5 +1,4 @@ wheel setuptools < 80.0.0 -setuptools-scm < 8.0.0 pip >= 23.3,< 24.0 ; python_version < '3.12' pip >24 ; python_version >= '3.12' diff --git a/salt/channel/server.py b/salt/channel/server.py index 2b9f4f76d4bb..0872dda89cc5 100644 --- a/salt/channel/server.py +++ b/salt/channel/server.py @@ -1043,7 +1043,7 @@ def _publish_daemon(self, **kwargs): ) os.nice(self.opts["event_publisher_niceness"]) self.io_loop = tornado.ioloop.IOLoop.current() - tcp_master_pool_port = 4520 + tcp_master_pool_port = self.opts["cluster_pool_port"] self.pushers = [] self.auth_errors = {} for peer in self.opts.get("cluster_peers", []): diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 1e44215f6bcb..5129ea23c8e3 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -196,6 +196,8 @@ def _gather_buffer_space(): # to define where minion keys and the cluster private key will be # stored. "cluster_pki_dir": str, + # The port required to be open for a master cluster to properly function + "cluster_pool_port": int, # Use a module function to determine the unique identifier. If this is # set and 'id' is not set, it will allow invocation of a module function # to determine the value of 'id'. For simple invocations without function @@ -1668,6 +1670,7 @@ def _gather_buffer_space(): "cluster_id": None, "cluster_peers": [], "cluster_pki_dir": None, + "cluster_pool_port": 4520, "features": {}, "publish_signing_algorithm": "PKCS1v15-SHA1", } @@ -4103,7 +4106,7 @@ def apply_master_config(overrides=None, defaults=None): prepend_root_dir(opts, prepend_root_dirs) - # When a cluster id is defined, make sure the other nessicery bits a + # When a cluster id is defined, make sure the other necessary bits are # defined. if "cluster_id" not in opts: opts["cluster_id"] = None @@ -4121,7 +4124,7 @@ def apply_master_config(overrides=None, defaults=None): log.warning("Cluster peers defined without a cluster_id, ignoring.") opts["cluster_peers"] = [] if opts.get("cluster_pki_dir", None): - log.warning("Cluster pki defined without a cluster_id, ignoring.") + log.warning("Cluster pki dir defined without a cluster_id, ignoring.") opts["cluster_pki_dir"] = None # Enabling open mode requires that the value be set to True, and diff --git a/salt/master.py b/salt/master.py index c03f5917e217..13922b3a7ce8 100644 --- a/salt/master.py +++ b/salt/master.py @@ -386,7 +386,7 @@ def handle_key_rotate(self, now, drop_file_wait=5): self.opts["cachedir"], self.opts["user"], self.opts["id"] ) # There is currently no concept of a leader in a master - # cluster. Lets fake it till we make it with a little + # cluster. Let's fake it till we make it with a little # waiting period. time.sleep(drop_file_wait) to_rotate = ( diff --git a/tests/pytests/integration/cli/conftest.py b/tests/pytests/integration/cli/conftest.py index 1f87f17e58a9..48554dda4d8c 100644 --- a/tests/pytests/integration/cli/conftest.py +++ b/tests/pytests/integration/cli/conftest.py @@ -2,7 +2,6 @@ @pytest.fixture(scope="package") -@pytest.mark.core_test def salt_eauth_account(salt_eauth_account_factory): with salt_eauth_account_factory as account: yield account diff --git a/tests/pytests/integration/pillar/test_fileclient.py b/tests/pytests/integration/pillar/test_fileclient.py index 2fe4dc40b79c..0d35e249f762 100644 --- a/tests/pytests/integration/pillar/test_fileclient.py +++ b/tests/pytests/integration/pillar/test_fileclient.py @@ -4,21 +4,20 @@ def test_pillar_using_cp_module(salt_master, salt_minion, salt_cli, tmp_path): "*": - my_pillar """ - my_pillar = """ - {{% set file_content = salt.cp.get_file_str("{}") %}} - """.format( - str(tmp_path / "myfile.txt") - ) - my_file = """ - foobar + my_file = tmp_path / "my_file.txt" + my_file.write_text("foobar") + my_pillar = f""" + {{%- set something = salt['cp.get_file_str']("{str(my_file)}") %}} + file_content: {{{{ something }}}} """ - (tmp_path / "myfile.txt").write_text(my_file) with salt_master.pillar_tree.base.temp_file("top.sls", pillar_top): with salt_master.pillar_tree.base.temp_file("my_pillar.sls", my_pillar): - with salt_master.pillar_tree.base.temp_file("my_pillar.sls", my_pillar): - ret = salt_cli.run("state.apply", minion_tgt=salt_minion.id) - assert ret.returncode == 1 - assert ( - ret.json["no_|-states_|-states_|-None"]["comment"] - == "No states found for this minion" - ) + + ret = salt_cli.run("saltutil.pillar_refresh", minion_tgt=salt_minion.id) + assert ret.returncode == 0 + + pillar_ret = salt_cli.run( + "pillar.item", "file_content", minion_tgt=salt_minion.id + ) + assert pillar_ret.returncode == 0 + assert '"file_content": "foobar"' in pillar_ret.stdout diff --git a/tests/pytests/integration/pillar/test_httpclient_in_pillar.py b/tests/pytests/integration/pillar/test_httpclient_in_pillar.py index 666e51de52c4..9fbfa1d2c4c2 100644 --- a/tests/pytests/integration/pillar/test_httpclient_in_pillar.py +++ b/tests/pytests/integration/pillar/test_httpclient_in_pillar.py @@ -1,4 +1,4 @@ -def test_pillar_using_http_query(salt_master, salt_minion, salt_cli, tmp_path): +def test_pillar_using_http_query(salt_master, salt_minion, salt_cli): pillar_top = """ base: "*": @@ -11,19 +11,13 @@ def test_pillar_using_http_query(salt_master, salt_minion, salt_cli, tmp_path): with salt_master.pillar_tree.base.temp_file("top.sls", pillar_top): with salt_master.pillar_tree.base.temp_file("http_pillar_test.sls", my_pillar): - with salt_master.pillar_tree.base.temp_file( - "http_pillar_test.sls", my_pillar - ): - ret = salt_cli.run("state.apply", minion_tgt=salt_minion.id) - assert ret.returncode == 1 - assert ( - ret.data["no_|-states_|-states_|-None"]["comment"] - == "No states found for this minion" - ) - pillar_ret = salt_cli.run( - "pillar.item", "http_query_test", minion_tgt=salt_minion.id - ) - assert pillar_ret.returncode == 0 + # We may need this for the following pillar.item to work + ret = salt_cli.run("saltutil.pillar_refresh", minion_tgt=salt_minion.id) + assert ret.returncode == 0 - assert '"http_query_test": 200' in pillar_ret.stdout + pillar_ret = salt_cli.run( + "pillar.item", "http_query_test", minion_tgt=salt_minion.id + ) + assert pillar_ret.returncode == 0 + assert '"http_query_test": 200' in pillar_ret.stdout diff --git a/tests/pytests/unit/config/test_master_config.py b/tests/pytests/unit/config/test_master_config.py index 5de8cd18c9fd..0cf3adcb88e7 100644 --- a/tests/pytests/unit/config/test_master_config.py +++ b/tests/pytests/unit/config/test_master_config.py @@ -13,6 +13,8 @@ def test_apply_no_cluster_id(): assert opts["cluster_id"] is None assert "cluster_pki_dir" in opts assert opts["cluster_pki_dir"] is None + assert "cluster_pool_port" in opts + assert opts["cluster_pool_port"] == 4520 def test_apply_default_for_cluster(): @@ -33,6 +35,10 @@ def test_apply_default_for_cluster(): assert "cluster_peers" in opts assert [] == opts["cluster_peers"] + # the cluster pool port defaults to 4520 + assert "cluster_pool_port" in opts + assert opts["cluster_pool_port"] == 4520 + def test_apply_for_cluster(): defaults = salt.config.DEFAULT_MASTER_OPTS.copy() @@ -46,6 +52,7 @@ def test_apply_for_cluster(): "127.0.0.3", ], "cluster_pki_dir": cluster_dir, + "cluster_pool_port": 5500, } opts = salt.config.apply_master_config(overrides, defaults) @@ -56,6 +63,10 @@ def test_apply_for_cluster(): assert "cluster_pki_dir" in opts assert cluster_dir == opts["cluster_pki_dir"] + # the cluster pool port defaults to 4520 + assert "cluster_pool_port" in opts + assert opts["cluster_pool_port"] == 5500 + # the cluster peers defaults to empty list assert "cluster_peers" in opts assert isinstance(opts["cluster_peers"], list)