Merge branch 'main' into smoke_tests_cleanup

Author: DaanHoogland

Diff for: .asf.yaml

@@ -27,15 +27,21 @@ github:
     - infrastructure
     - java
     - python
-    - hosting
     - kvm
+    - libvirt
     - vsphere
+    - vmware
     - xenserver
     - xcp-ng
+    - orchestration
+    - virtualization
+    - virtual-machine
+    - kubernetes
 
   features:
     wiki: true
     issues: true
+    discussions: true
     projects: true
 
   enabled_merge_buttons:
@@ -45,5 +51,19 @@ github:
 
   collaborators:
     - acs-robot
+    - rajujith
+    - GaOrtiga
+    - SadiJr
+    - winterhazel
+    - gpordeus
+    - hsato03
+    - bernardodemarco
+    - abh1sar
 
   protected_branches: ~
+
+notifications:
+  commits:      [email protected]
+  issues:       [email protected]
+  pullrequests: [email protected]
+  discussions:  [email protected]

Diff for: .github/linters/.flake8

@@ -15,11 +15,17 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# E112 Expected an indented block
+# E113 Unexpected indentation
+# E133 Closing bracket is missing indentation
 # E223 Tab before operator
 # E224 Tab after operator
+# E227 Missing whitespace around bitwise or shift operator
 # E242 Tab after ','
 # E273 Tab after keyword
 # E274 Tab before keyword
+# E742 Do not define classes named 'I', 'O', or 'l'
+# E743 Do not define functions named 'I', 'O', or 'l'
 # E901 SyntaxError or IndentationError
 # E902 IOError
 # W291 Trailing whitespace
@@ -28,4 +34,7 @@
 # W391 Blank line at end of file
 
 [flake8]
-select = E223,E224,E242,E273,E274,E901,E902,W291,W292,W293,W391
+exclude =
+    .git,
+    venv
+select = E112,E113,E133,E223,E224,E227,E242,E273,E274,E742,E743,E901,E902,W291,W292,W293,W391

Diff for: .github/linters/.markdown-lint.yml

@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# MD001/heading-increment Heading levels should only increment by one level at a time
+MD001: false
+
+# MD003/heading-style Heading style
+MD003: false
+
+# MD004/ul-style Unordered list style
+MD004: false
+
+# MD007/ul-indent Unordered list indentation
+MD007: false
+
+# MD009/no-trailing-spaces Trailing spaces
+MD009: false
+
+# MD010/no-hard-tabs Hard tabs
+MD010: false
+
+# MD012/no-multiple-blanks Multiple consecutive blank lines
+MD012: false
+
+# MD013/line-length Line length
+MD013: false
+
+# MD014/commands-show-output Dollar signs used before commands without showing output
+MD014: false
+
+# MD018/no-missing-space-atx No space after hash on atx style heading
+MD018: false
+
+# MD019/no-multiple-space-atx Multiple spaces after hash on atx style heading
+MD019: false
+
+# MD022/blanks-around-headings Headings should be surrounded by blank lines
+MD022: false
+
+# MD023/heading-start-left Headings must start at the beginning of the line
+MD023: false
+
+# MD024/no-duplicate-heading Multiple headings with the same content
+MD024: false
+
+# MD025/single-title/single-h1 Multiple top-level headings in the same document
+MD025: false
+
+# MD026/no-trailing-punctuation Trailing punctuation in heading
+MD026: false
+
+# MD028/no-blanks-blockquote Blank line inside blockquote
+MD028: false
+
+# MD029/ol-prefix Ordered list item prefix
+MD029: false
+
+# MD031/blanks-around-fences Fenced code blocks should be surrounded by blank lines
+MD031: false
+
+# MD032/blanks-around-lists Lists should be surrounded by blank lines
+MD032: false
+
+# MD033/no-inline-html Inline HTML
+MD033: false
+
+# MD034/no-bare-urls Bare URL used
+MD034: false
+
+# MD036/no-emphasis-as-heading Emphasis used instead of a heading
+MD036: false
+
+# MD037/no-space-in-emphasis Spaces inside emphasis markers
+MD037: false
+
+# MD040/fenced-code-language Fenced code blocks should have a language specified
+MD040: false
+
+# MD041/first-line-heading/first-line-h1 First line in a file should be a top-level heading
+MD041: false
+
+# MD046/code-block-style Code block style
+MD046: false
+
+# MD052/reference-links-images Reference links and images should use a label that is defined
+MD052: false

Diff for: .github/workflows/build.yml

@@ -23,11 +23,14 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up JDK 11
         uses: actions/setup-java@v4
@@ -38,7 +41,7 @@ jobs:
           cache: maven
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.8'
           architecture: 'x64'

Diff for: .github/workflows/ci.yml

@@ -23,6 +23,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     if: github.repository == 'apache/cloudstack'
@@ -51,6 +54,7 @@ jobs:
                   smoke/test_deploy_vm_with_userdata
                   smoke/test_deploy_vms_in_parallel
                   smoke/test_deploy_vms_with_varied_deploymentplanners
+                  smoke/test_restore_vm
                   smoke/test_diagnostics
                   smoke/test_direct_download
                   smoke/test_disk_offerings
@@ -59,7 +63,8 @@ jobs:
                   smoke/test_domain_network_offerings
                   smoke/test_domain_service_offerings
                   smoke/test_domain_vpc_offerings",
-                "smoke/test_dynamicroles
+                "smoke/test_cluster_drs
+                  smoke/test_dynamicroles
                   smoke/test_enable_account_settings_for_domain
                   smoke/test_enable_role_based_users_in_projects
                   smoke/test_events_resource
@@ -80,14 +85,16 @@ jobs:
                   smoke/test_metrics_api
                   smoke/test_migration
                   smoke/test_multipleips_per_nic
-                  smoke/test_nested_virtualization",
+                  smoke/test_nested_virtualization
+                  smoke/test_set_sourcenat",
                 "smoke/test_network
                   smoke/test_network_acl
                   smoke/test_network_ipv6
                   smoke/test_network_permissions
                   smoke/test_nic
                   smoke/test_nic_adapter_type
                   smoke/test_non_contigiousvlan
+                  smoke/test_object_stores
                   smoke/test_outofbandmanagement
                   smoke/test_outofbandmanagement_nestedplugin
                   smoke/test_over_provisioning
@@ -106,7 +113,8 @@ jobs:
                   smoke/test_reset_configuration_settings
                   smoke/test_reset_vm_on_reboot
                   smoke/test_resource_accounting
-                  smoke/test_resource_detail",
+                  smoke/test_resource_detail
+                  smoke/test_global_acls",
                 "smoke/test_router_dhcphosts
                   smoke/test_router_dns
                   smoke/test_router_dnsservice
@@ -124,6 +132,7 @@ jobs:
                   smoke/test_usage
                   smoke/test_usage_events
                   smoke/test_vm_deployment_planner
+                  smoke/test_vm_schedule
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import
                   smoke/test_vm_snapshot_kvm
@@ -171,7 +180,8 @@ jobs:
                 "component/test_project_usage
                   component/test_protocol_number_security_group
                   component/test_public_ip
-                  component/test_resource_limits",
+                  component/test_resource_limits
+                  component/test_resource_limit_tags",
                 "component/test_regions_accounts
                   component/test_routers
                   component/test_snapshots
@@ -186,11 +196,20 @@ jobs:
                   component/test_vpc_network
                   component/test_vpc_offerings
                   component/test_vpc_routers
-                  component/test_vpn_users",
-                "component/test_vpc_network_lbrules" ]
+                  component/test_vpn_users
+                  component/test_vpc_network_lbrules",
+                "smoke/test_list_accounts
+                  smoke/test_list_disk_offerings
+                  smoke/test_list_domains
+                  smoke/test_list_hosts
+                  smoke/test_list_service_offerings
+                  smoke/test_list_storage_pools
+                  smoke/test_list_volumes"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up JDK
         uses: actions/setup-java@v4
@@ -201,7 +220,7 @@ jobs:
           cache: maven
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.8'
           architecture: 'x64'
@@ -215,6 +234,11 @@ jobs:
         run: |
           python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid
 
+      - name: Install jacoco dependencies
+        run: |
+          wget https://github.com/jacoco/jacoco/releases/download/v0.8.10/jacoco-0.8.10.zip
+          unzip jacoco-0.8.10.zip -d jacoco
+
       - name: Env details
         run: |
           uname -a
@@ -247,16 +271,23 @@ jobs:
           mvn -q -Pdeveloper -pl developer -Ddeploydb
           mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator
 
+      - name: Generate jacoco-coverage.sh
+        run: |
+          echo "java -jar jacoco/lib/jacococli.jar report jacoco-it.exec \\" > jacoco-report.sh
+          find . | grep "target/classes" | sed 's/\/classes\//\/classes /g' | awk '{print "--classfiles", $1, "\\"}' | sort |uniq >> jacoco-report.sh
+          find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh
+          echo "--xml jacoco-coverage.xml" >> jacoco-report.sh
+
       - name: Start CloudStack Management Server with Simulator
         run: |
-          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom"
+          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom  -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver"
           echo -e "\nStarting simulator"
           set +e
           mvn -Dsimulator -Dorg.eclipse.jetty.annotations.maxWait=120 -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log || true &
           while ! nc -vzw 5 localhost 8096 2>&1 > /dev/null; do grep Exception /tmp/jetty-log; sleep 10; done
           set -e
           echo -e "\nStarting Advanced Zone DataCenter deployment"
-          python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advanced.cfg 2>&1 || true
+          python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advdualzone.cfg 2>&1 || true
 
       - name: Run Integration Tests with Simulator
         run: |
@@ -269,14 +300,25 @@ jobs:
           TESTS=($(echo $TESTS | tr -d '\n' | tr -s ' '))
           for suite in "${TESTS[@]}" ; do
             echo -e "Currently running test: $suite\n"
-            time nosetests-3.4 --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advanced.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=Sandbox-simulator --hypervisor=simulator || true ;
+            time nosetests-3.4 --with-xunit --xunit-file=integration-test-results/$suite.xml --with-marvin --marvin-config=setup/dev/advdualzone.cfg test/integration/$suite.py -s -a tags=advanced,required_hardware=false --zone=zim1 --hypervisor=simulator || true ;
           done
 
           echo -e "Stopping Simulator, integration tests run completed\n"
+          java -jar jacoco/lib/jacococli.jar dump --address localhost --port 36320 --destfile jacoco-it.exec
+          bash jacoco-report.sh
           mvn -Dsimulator -pl client jetty:stop 2>&1
           find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \;
 
       - name: Integration Tests Result
         run: |
           echo -e "Simulator CI Test Results: (only failures listed)\n"
           python3 ./tools/marvin/xunit-reader.py integration-test-results/
+
+      - uses: codecov/codecov-action@v4
+        with:
+          files: jacoco-coverage.xml
+          fail_ci_if_error: true
+          flags: simulator-marvin-tests
+          verbose: true
+          name: codecov
+          token: ${{ secrets.CODECOV_TOKEN }}

Diff for: .github/workflows/codecov.yml

@@ -32,7 +32,9 @@ jobs:
     name: codecov
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up JDK11
         uses: actions/setup-java@v4

Diff for: .github/workflows/docker-cloudstack-simulator.yml

@@ -47,7 +47,7 @@ jobs:
       - name: Set Docker repository name
         run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set ACS version
         run: echo "ACS_VERSION=$(grep '<version>' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV