From 02da4766255463ad68a0e0ad5f5908cbde68d99c Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:06:19 +0200 Subject: [PATCH 01/14] Move all config in conf --- .sonarcloud.properties | 9 ---- .flake8 => conf/.flake8 | 0 deploy.bat => conf/deploy.bat | 0 deploy.sh => conf/deploy.sh | 0 pylintrc => conf/pylintrc | 0 release.Dockerfile => conf/release.Dockerfile | 0 run_linters.sh => conf/run_linters.sh | 8 +++- run_tests.sh => conf/run_tests.sh | 0 scan.sh => conf/scan.sh | 0 .../shellcheck2sonar.py | 0 .../snapshot.Dockerfile | 0 .../sonar-project.properties | 0 migration.Dockerfile | 42 ------------------- 13 files changed, 6 insertions(+), 53 deletions(-) delete mode 100644 .sonarcloud.properties rename .flake8 => conf/.flake8 (100%) rename deploy.bat => conf/deploy.bat (100%) rename deploy.sh => conf/deploy.sh (100%) rename pylintrc => conf/pylintrc (100%) rename release.Dockerfile => conf/release.Dockerfile (100%) rename run_linters.sh => conf/run_linters.sh (83%) rename run_tests.sh => conf/run_tests.sh (100%) rename scan.sh => conf/scan.sh (100%) rename shellcheck2sonar.py => conf/shellcheck2sonar.py (100%) rename snapshot.Dockerfile => conf/snapshot.Dockerfile (100%) rename sonar-project.properties => conf/sonar-project.properties (100%) delete mode 100644 migration.Dockerfile diff --git a/.sonarcloud.properties b/.sonarcloud.properties deleted file mode 100644 index 248db4465..000000000 --- a/.sonarcloud.properties +++ /dev/null @@ -1,9 +0,0 @@ -# Required metadata -# sonar.projectVersion=1.0 - -# Comma-separated paths to directories with sources (required) -sonar.sources=. -# Encoding of the source files -sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/*.csv,**/*.xls* -sonar.python.version=3.6 \ No newline at end of file diff --git a/.flake8 b/conf/.flake8 similarity index 100% rename from .flake8 rename to conf/.flake8 diff --git a/deploy.bat b/conf/deploy.bat similarity index 100% rename from deploy.bat rename to conf/deploy.bat diff --git a/deploy.sh b/conf/deploy.sh similarity index 100% rename from deploy.sh rename to conf/deploy.sh diff --git a/pylintrc b/conf/pylintrc similarity index 100% rename from pylintrc rename to conf/pylintrc diff --git a/release.Dockerfile b/conf/release.Dockerfile similarity index 100% rename from release.Dockerfile rename to conf/release.Dockerfile diff --git a/run_linters.sh b/conf/run_linters.sh similarity index 83% rename from run_linters.sh rename to conf/run_linters.sh index 306ab1009..d4949bc4d 100755 --- a/run_linters.sh +++ b/conf/run_linters.sh @@ -19,7 +19,11 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # -buildDir="build" +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +buildDir="$ROOTDIR/build" pylintReport="$buildDir/pylint-report.out" # banditReport="$buildDir/bandit-report.json" flake8Report="$buildDir/flake8-report.out" @@ -29,7 +33,7 @@ externalIssueReport="$buildDir/external-issue-report.json" echo "Running pylint" rm -f $pylintReport -pylint ./*.py ./*/*.py -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" | tee $pylintReport +pylint --rcfile $CONFDIR/pylintrc ./*.py ./*/*.py -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" | tee $pylintReport re=$? if [ "$re" == "32" ]; then >&2 echo "ERROR: pylint execution failed, errcode $re, aborting..." diff --git a/run_tests.sh b/conf/run_tests.sh similarity index 100% rename from run_tests.sh rename to conf/run_tests.sh diff --git a/scan.sh b/conf/scan.sh similarity index 100% rename from scan.sh rename to conf/scan.sh diff --git a/shellcheck2sonar.py b/conf/shellcheck2sonar.py similarity index 100% rename from shellcheck2sonar.py rename to conf/shellcheck2sonar.py diff --git a/snapshot.Dockerfile b/conf/snapshot.Dockerfile similarity index 100% rename from snapshot.Dockerfile rename to conf/snapshot.Dockerfile diff --git a/sonar-project.properties b/conf/sonar-project.properties similarity index 100% rename from sonar-project.properties rename to conf/sonar-project.properties diff --git a/migration.Dockerfile b/migration.Dockerfile deleted file mode 100644 index 35cb679c8..000000000 --- a/migration.Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -FROM alpine:3.20.3 -LABEL maintainer="olivier.korach@gmail.com" -ENV IN_DOCKER="Yes" - -ARG USERNAME=sonar -ARG USER_UID=1000 -ARG GROUPNAME=sonar - -# Create the user -RUN addgroup -S ${GROUPNAME} && adduser -u ${USER_UID} -S ${USERNAME} -G ${GROUPNAME} - -# Install python/pip -ENV PYTHONUNBUFFERED=1 -RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python - -# create a virtual environment and add it to PATH so that it is -# applied for all future RUN and CMD calls -ENV VIRTUAL_ENV=/opt/venv -RUN python3 -m venv ${VIRTUAL_ENV} -ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" - -WORKDIR /opt/sonar-migration - -COPY ./sonar sonar -COPY ./requirements.txt . -COPY ./cli cli -COPY ./setup_migration.py . -COPY ./sonar_migration . -COPY ./README.md . -COPY ./LICENSE . -COPY ./sonar/audit sonar/audit - -RUN pip install --upgrade pip \ -&& pip install --no-cache-dir -r requirements.txt \ -&& pip install --no-cache-dir --upgrade pip setuptools wheel \ -&& python setup_migration.py bdist_wheel \ -&& pip install dist/sonar_migration-py3-*.whl --force-reinstall - -USER ${USERNAME} -WORKDIR /home/${USERNAME} - -ENTRYPOINT ["sonar-migration"] From 176fcde18a8b30ad86ca2a151945cfdbc4df8e2c Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:07:07 +0200 Subject: [PATCH 02/14] Move api-doc in doc --- {api-doc => doc/api}/Makefile | 0 {api-doc => doc/api}/make.bat | 0 {api-doc => doc/api}/source/aggregation_permissions.rst | 0 {api-doc => doc/api}/source/aggregations.rst | 0 {api-doc => doc/api}/source/application_permissions.rst | 0 {api-doc => doc/api}/source/applications.rst | 0 {api-doc => doc/api}/source/branches.rst | 0 {api-doc => doc/api}/source/components.rst | 0 {api-doc => doc/api}/source/conf.py | 0 {api-doc => doc/api}/source/devops.rst | 0 {api-doc => doc/api}/source/findings.rst | 0 {api-doc => doc/api}/source/global_permissions.rst | 0 {api-doc => doc/api}/source/groups.rst | 0 {api-doc => doc/api}/source/hotspots.rst | 0 {api-doc => doc/api}/source/index.rst | 0 {api-doc => doc/api}/source/issues.rst | 0 {api-doc => doc/api}/source/languages.rst | 0 {api-doc => doc/api}/source/measures.rst | 0 {api-doc => doc/api}/source/metrics.rst | 0 {api-doc => doc/api}/source/organizations.rst | 0 {api-doc => doc/api}/source/permission_templates.rst | 0 {api-doc => doc/api}/source/permissions.rst | 0 {api-doc => doc/api}/source/platform.rst | 0 {api-doc => doc/api}/source/portfolio_permissions.rst | 0 {api-doc => doc/api}/source/portfolios.rst | 0 {api-doc => doc/api}/source/project_permissions.rst | 0 {api-doc => doc/api}/source/projects.rst | 0 {api-doc => doc/api}/source/pullrequests.rst | 0 {api-doc => doc/api}/source/quality_permissions.rst | 0 {api-doc => doc/api}/source/qualitygate_permissions.rst | 0 {api-doc => doc/api}/source/qualitygates.rst | 0 {api-doc => doc/api}/source/qualityprofile_permissions.rst | 0 {api-doc => doc/api}/source/qualityprofiles.rst | 0 {api-doc => doc/api}/source/rules.rst | 0 {api-doc => doc/api}/source/sq_object.rst | 0 {api-doc => doc/api}/source/tasks.rst | 0 {api-doc => doc/api}/source/tokens.rst | 0 {api-doc => doc/api}/source/users.rst | 0 {api-doc => doc/api}/source/webhooks.rst | 0 39 files changed, 0 insertions(+), 0 deletions(-) rename {api-doc => doc/api}/Makefile (100%) rename {api-doc => doc/api}/make.bat (100%) rename {api-doc => doc/api}/source/aggregation_permissions.rst (100%) rename {api-doc => doc/api}/source/aggregations.rst (100%) rename {api-doc => doc/api}/source/application_permissions.rst (100%) rename {api-doc => doc/api}/source/applications.rst (100%) rename {api-doc => doc/api}/source/branches.rst (100%) rename {api-doc => doc/api}/source/components.rst (100%) rename {api-doc => doc/api}/source/conf.py (100%) rename {api-doc => doc/api}/source/devops.rst (100%) rename {api-doc => doc/api}/source/findings.rst (100%) rename {api-doc => doc/api}/source/global_permissions.rst (100%) rename {api-doc => doc/api}/source/groups.rst (100%) rename {api-doc => doc/api}/source/hotspots.rst (100%) rename {api-doc => doc/api}/source/index.rst (100%) rename {api-doc => doc/api}/source/issues.rst (100%) rename {api-doc => doc/api}/source/languages.rst (100%) rename {api-doc => doc/api}/source/measures.rst (100%) rename {api-doc => doc/api}/source/metrics.rst (100%) rename {api-doc => doc/api}/source/organizations.rst (100%) rename {api-doc => doc/api}/source/permission_templates.rst (100%) rename {api-doc => doc/api}/source/permissions.rst (100%) rename {api-doc => doc/api}/source/platform.rst (100%) rename {api-doc => doc/api}/source/portfolio_permissions.rst (100%) rename {api-doc => doc/api}/source/portfolios.rst (100%) rename {api-doc => doc/api}/source/project_permissions.rst (100%) rename {api-doc => doc/api}/source/projects.rst (100%) rename {api-doc => doc/api}/source/pullrequests.rst (100%) rename {api-doc => doc/api}/source/quality_permissions.rst (100%) rename {api-doc => doc/api}/source/qualitygate_permissions.rst (100%) rename {api-doc => doc/api}/source/qualitygates.rst (100%) rename {api-doc => doc/api}/source/qualityprofile_permissions.rst (100%) rename {api-doc => doc/api}/source/qualityprofiles.rst (100%) rename {api-doc => doc/api}/source/rules.rst (100%) rename {api-doc => doc/api}/source/sq_object.rst (100%) rename {api-doc => doc/api}/source/tasks.rst (100%) rename {api-doc => doc/api}/source/tokens.rst (100%) rename {api-doc => doc/api}/source/users.rst (100%) rename {api-doc => doc/api}/source/webhooks.rst (100%) diff --git a/api-doc/Makefile b/doc/api/Makefile similarity index 100% rename from api-doc/Makefile rename to doc/api/Makefile diff --git a/api-doc/make.bat b/doc/api/make.bat similarity index 100% rename from api-doc/make.bat rename to doc/api/make.bat diff --git a/api-doc/source/aggregation_permissions.rst b/doc/api/source/aggregation_permissions.rst similarity index 100% rename from api-doc/source/aggregation_permissions.rst rename to doc/api/source/aggregation_permissions.rst diff --git a/api-doc/source/aggregations.rst b/doc/api/source/aggregations.rst similarity index 100% rename from api-doc/source/aggregations.rst rename to doc/api/source/aggregations.rst diff --git a/api-doc/source/application_permissions.rst b/doc/api/source/application_permissions.rst similarity index 100% rename from api-doc/source/application_permissions.rst rename to doc/api/source/application_permissions.rst diff --git a/api-doc/source/applications.rst b/doc/api/source/applications.rst similarity index 100% rename from api-doc/source/applications.rst rename to doc/api/source/applications.rst diff --git a/api-doc/source/branches.rst b/doc/api/source/branches.rst similarity index 100% rename from api-doc/source/branches.rst rename to doc/api/source/branches.rst diff --git a/api-doc/source/components.rst b/doc/api/source/components.rst similarity index 100% rename from api-doc/source/components.rst rename to doc/api/source/components.rst diff --git a/api-doc/source/conf.py b/doc/api/source/conf.py similarity index 100% rename from api-doc/source/conf.py rename to doc/api/source/conf.py diff --git a/api-doc/source/devops.rst b/doc/api/source/devops.rst similarity index 100% rename from api-doc/source/devops.rst rename to doc/api/source/devops.rst diff --git a/api-doc/source/findings.rst b/doc/api/source/findings.rst similarity index 100% rename from api-doc/source/findings.rst rename to doc/api/source/findings.rst diff --git a/api-doc/source/global_permissions.rst b/doc/api/source/global_permissions.rst similarity index 100% rename from api-doc/source/global_permissions.rst rename to doc/api/source/global_permissions.rst diff --git a/api-doc/source/groups.rst b/doc/api/source/groups.rst similarity index 100% rename from api-doc/source/groups.rst rename to doc/api/source/groups.rst diff --git a/api-doc/source/hotspots.rst b/doc/api/source/hotspots.rst similarity index 100% rename from api-doc/source/hotspots.rst rename to doc/api/source/hotspots.rst diff --git a/api-doc/source/index.rst b/doc/api/source/index.rst similarity index 100% rename from api-doc/source/index.rst rename to doc/api/source/index.rst diff --git a/api-doc/source/issues.rst b/doc/api/source/issues.rst similarity index 100% rename from api-doc/source/issues.rst rename to doc/api/source/issues.rst diff --git a/api-doc/source/languages.rst b/doc/api/source/languages.rst similarity index 100% rename from api-doc/source/languages.rst rename to doc/api/source/languages.rst diff --git a/api-doc/source/measures.rst b/doc/api/source/measures.rst similarity index 100% rename from api-doc/source/measures.rst rename to doc/api/source/measures.rst diff --git a/api-doc/source/metrics.rst b/doc/api/source/metrics.rst similarity index 100% rename from api-doc/source/metrics.rst rename to doc/api/source/metrics.rst diff --git a/api-doc/source/organizations.rst b/doc/api/source/organizations.rst similarity index 100% rename from api-doc/source/organizations.rst rename to doc/api/source/organizations.rst diff --git a/api-doc/source/permission_templates.rst b/doc/api/source/permission_templates.rst similarity index 100% rename from api-doc/source/permission_templates.rst rename to doc/api/source/permission_templates.rst diff --git a/api-doc/source/permissions.rst b/doc/api/source/permissions.rst similarity index 100% rename from api-doc/source/permissions.rst rename to doc/api/source/permissions.rst diff --git a/api-doc/source/platform.rst b/doc/api/source/platform.rst similarity index 100% rename from api-doc/source/platform.rst rename to doc/api/source/platform.rst diff --git a/api-doc/source/portfolio_permissions.rst b/doc/api/source/portfolio_permissions.rst similarity index 100% rename from api-doc/source/portfolio_permissions.rst rename to doc/api/source/portfolio_permissions.rst diff --git a/api-doc/source/portfolios.rst b/doc/api/source/portfolios.rst similarity index 100% rename from api-doc/source/portfolios.rst rename to doc/api/source/portfolios.rst diff --git a/api-doc/source/project_permissions.rst b/doc/api/source/project_permissions.rst similarity index 100% rename from api-doc/source/project_permissions.rst rename to doc/api/source/project_permissions.rst diff --git a/api-doc/source/projects.rst b/doc/api/source/projects.rst similarity index 100% rename from api-doc/source/projects.rst rename to doc/api/source/projects.rst diff --git a/api-doc/source/pullrequests.rst b/doc/api/source/pullrequests.rst similarity index 100% rename from api-doc/source/pullrequests.rst rename to doc/api/source/pullrequests.rst diff --git a/api-doc/source/quality_permissions.rst b/doc/api/source/quality_permissions.rst similarity index 100% rename from api-doc/source/quality_permissions.rst rename to doc/api/source/quality_permissions.rst diff --git a/api-doc/source/qualitygate_permissions.rst b/doc/api/source/qualitygate_permissions.rst similarity index 100% rename from api-doc/source/qualitygate_permissions.rst rename to doc/api/source/qualitygate_permissions.rst diff --git a/api-doc/source/qualitygates.rst b/doc/api/source/qualitygates.rst similarity index 100% rename from api-doc/source/qualitygates.rst rename to doc/api/source/qualitygates.rst diff --git a/api-doc/source/qualityprofile_permissions.rst b/doc/api/source/qualityprofile_permissions.rst similarity index 100% rename from api-doc/source/qualityprofile_permissions.rst rename to doc/api/source/qualityprofile_permissions.rst diff --git a/api-doc/source/qualityprofiles.rst b/doc/api/source/qualityprofiles.rst similarity index 100% rename from api-doc/source/qualityprofiles.rst rename to doc/api/source/qualityprofiles.rst diff --git a/api-doc/source/rules.rst b/doc/api/source/rules.rst similarity index 100% rename from api-doc/source/rules.rst rename to doc/api/source/rules.rst diff --git a/api-doc/source/sq_object.rst b/doc/api/source/sq_object.rst similarity index 100% rename from api-doc/source/sq_object.rst rename to doc/api/source/sq_object.rst diff --git a/api-doc/source/tasks.rst b/doc/api/source/tasks.rst similarity index 100% rename from api-doc/source/tasks.rst rename to doc/api/source/tasks.rst diff --git a/api-doc/source/tokens.rst b/doc/api/source/tokens.rst similarity index 100% rename from api-doc/source/tokens.rst rename to doc/api/source/tokens.rst diff --git a/api-doc/source/users.rst b/doc/api/source/users.rst similarity index 100% rename from api-doc/source/users.rst rename to doc/api/source/users.rst diff --git a/api-doc/source/webhooks.rst b/doc/api/source/webhooks.rst similarity index 100% rename from api-doc/source/webhooks.rst rename to doc/api/source/webhooks.rst From 1e8c99817ec18bd15c49531b07715d6a5986c8dc Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:11:13 +0200 Subject: [PATCH 03/14] Set conf config file for linters --- conf/run_linters.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/run_linters.sh b/conf/run_linters.sh index d4949bc4d..e7300d911 100755 --- a/conf/run_linters.sh +++ b/conf/run_linters.sh @@ -33,7 +33,7 @@ externalIssueReport="$buildDir/external-issue-report.json" echo "Running pylint" rm -f $pylintReport -pylint --rcfile $CONFDIR/pylintrc ./*.py ./*/*.py -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" | tee $pylintReport +pylint --rcfile $CONFDIR/pylintrc $ROOTDIR/*.py $ROOTDIR/*/*.py -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" | tee $pylintReport re=$? if [ "$re" == "32" ]; then >&2 echo "ERROR: pylint execution failed, errcode $re, aborting..." @@ -43,11 +43,11 @@ fi echo "Running flake8" rm -f $flake8Report # See .flake8 file for settings -flake8 . >$flake8Report +flake8 --config "$CONFIG/.flake8" "$ROOTDIR" >$flake8Report # echo "Running bandit" # rm -f $banditReport # bandit --exit-zero -f json --skip B311,B303,B101 -r . -x .vscode,./tests >$banditReport echo "Running shellcheck" -shellcheck ./*.sh ./*/*.sh -s bash -f json | ./shellcheck2sonar.py >$externalIssueReport +shellcheck $ROOTDIR/*.sh $ROOTDIR/*/*.sh -s bash -f json | $CONFDIR/shellcheck2sonar.py >$externalIssueReport From 96d35ca138f6b909fa7a33f51f9d50a9bffa1929 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:13:34 +0200 Subject: [PATCH 04/14] Make all runs relative to repo root --- conf/run_tests.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/conf/run_tests.sh b/conf/run_tests.sh index 2efe1bec9..ce7931e65 100755 --- a/conf/run_tests.sh +++ b/conf/run_tests.sh @@ -19,14 +19,18 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # -buildDir="build" +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +buildDir="$ROOTDIR/build" coverageReport="$buildDir/coverage.xml" [ ! -d $buildDir ] && mkdir $buildDir echo "Running tests" -chmod u-rw test/sif_not_readable.json +unreadableSif=$ROOTDIR/test/sif_not_readable.json +chmod u-rw $unreadableSif export SONAR_HOST_URL=${1:-${SONAR_HOST_URL_TEST}} -coverage run --source=. -m pytest test/ +coverage run --source=$ROOTDIR -m pytest $ROOTDIR/test/ coverage xml -o $coverageReport -chmod u+rw test/sif_not_readable.json +chmod u+rw $unreadableSif From 21d06b9bc3aba90dc8bd1265b7b69a038db268b5 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:15:50 +0200 Subject: [PATCH 05/14] Make scan relative to repo root --- conf/scan.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conf/scan.sh b/conf/scan.sh index 0ac37e681..c68e58ac4 100755 --- a/conf/scan.sh +++ b/conf/scan.sh @@ -19,6 +19,10 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + dolint=true dotest=false @@ -40,7 +44,7 @@ do shift done -buildDir="build" +buildDir="$ROOTDIR/build" pylintReport="$buildDir/pylint-report.out" banditReport="$buildDir/bandit-report.json" flake8Report="$buildDir/flake8-report.out" @@ -58,7 +62,7 @@ if [ "$dolint" != "false" ]; then ./run_linters.sh fi -version=$(grep PACKAGE_VERSION sonar/version.py | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//") +version=$(grep PACKAGE_VERSION $ROOTDIR/sonar/version.py | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//") cmd="sonar-scanner -Dsonar.projectVersion=$version \ -Dsonar.python.flake8.reportPaths=$flake8Report \ From 60f8079293b5620630dbbb1c21f10cd4b8c17baf Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:19:46 +0200 Subject: [PATCH 06/14] Specify properties file location in conf --- conf/scan.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/scan.sh b/conf/scan.sh index c68e58ac4..b18ed2951 100755 --- a/conf/scan.sh +++ b/conf/scan.sh @@ -65,6 +65,7 @@ fi version=$(grep PACKAGE_VERSION $ROOTDIR/sonar/version.py | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//") cmd="sonar-scanner -Dsonar.projectVersion=$version \ + -Dproject.settings=$CONFDIR/sonar-project.properties \ -Dsonar.python.flake8.reportPaths=$flake8Report \ -Dsonar.python.pylint.reportPaths=$pylintReport \ -Dsonar.python.bandit.reportPaths=$banditReport \ From b23e5ce54172ef96af5d9e5ae6ff94766b076781 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:20:23 +0200 Subject: [PATCH 07/14] Run everything all relative to repo root --- conf/deploy.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/conf/deploy.sh b/conf/deploy.sh index 1e23abe1c..3d4ea21f8 100755 --- a/conf/deploy.sh +++ b/conf/deploy.sh @@ -19,6 +19,10 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + build_docs=1 build_image=1 release=0 @@ -44,19 +48,19 @@ while [ $# -ne 0 ]; do done black --line-length=150 . -rm -rf build dist -python3 setup.py bdist_wheel +rm -rf $ROOTDIR/build $ROOTDIR/dist +python3 $ROOTDIR/setup.py bdist_wheel # Deploy locally for tests -pip install --upgrade --force-reinstall dist/sonar_tools-*-py3-*.whl +pip install --upgrade --force-reinstall $ROOTDIR/dist/sonar_tools-*-py3-*.whl if [ "$build_image" == "1" ]; then - docker build -t olivierkorach/sonar-tools:3.5 -t olivierkorach/sonar-tools:latest -f snapshot.Dockerfile . --load + docker build -t olivierkorach/sonar-tools:3.5 -t olivierkorach/sonar-tools:latest -f $CONFDIR/snapshot.Dockerfile $ROOTDIR --load fi if [ "$build_docs" == "1" ]; then - rm -rf api-doc/build - sphinx-build -b html api-doc/source api-doc/build + rm -rf doc/api/build + sphinx-build -b html doc/api/source doc/api/build fi # Deploy on pypi.org once released @@ -64,10 +68,10 @@ if [ "$release" = "1" ]; then echo "Confirm release [y/n] ?" read -r confirm if [ "$confirm" = "y" ]; then - python3 -m twine upload dist/sonar_tools-*-py3-*.whl + python3 -m twine upload $ROOTDIR/dist/sonar_tools-*-py3-*.whl fi fi if [ "$release_docker" = "1" ]; then - docker buildx build --push --platform linux/amd64,linux/arm64 -t olivierkorach/sonar-tools:3.5 -t olivierkorach/sonar-tools:latest -f release.Dockerfile . + docker buildx build --push --platform linux/amd64,linux/arm64 -t olivierkorach/sonar-tools:3.5 -t olivierkorach/sonar-tools:latest -f $ROOTDIR/release.Dockerfile . fi \ No newline at end of file From 9ad9b085a4d49d4ed9fba7f8d11b0d6c47492b77 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:22:29 +0200 Subject: [PATCH 08/14] Build everything from repo root --- migration/deploy.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/migration/deploy.sh b/migration/deploy.sh index 8f82a635c..a4a56f77e 100755 --- a/migration/deploy.sh +++ b/migration/deploy.sh @@ -19,6 +19,10 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + build_image=1 release=0 @@ -40,14 +44,14 @@ while [ $# -ne 0 ]; do done black --line-length=150 . -rm -rf build dist -python3 setup_migration.py bdist_wheel +rm -rf $ROOTDIR/build $ROOTDIR/dist +python3 $ROOTDIR/setup_migration.py bdist_wheel # Deploy locally for tests -pip install --upgrade --force-reinstall dist/sonar_migration-*-py3-*.whl +pip install --upgrade --force-reinstall $ROOTDIR/dist/sonar_migration-*-py3-*.whl if [ "$build_image" == "1" ]; then - docker build -t olivierkorach/sonar-migration:latest -f migration/snapshot.Dockerfile . --load + docker build -t olivierkorach/sonar-migration:latest -f migration/snapshot.Dockerfile $ROOTDIR --load fi # Deploy on pypi.org once released @@ -55,10 +59,10 @@ if [ "$release" = "1" ]; then echo "Confirm release [y/n] ?" read -r confirm if [ "$confirm" = "y" ]; then - python3 -m twine upload dist/sonar_migration-*-py3-*.whl + python3 -m twine upload $ROOTDIR/dist/sonar_migration-*-py3-*.whl fi fi if [ "$release_docker" = "1" ]; then - docker buildx build --push --platform linux/amd64,linux/arm64 -t olivierkorach/sonar-migration:0.3 -t olivierkorach/sonar-migration:latest -f migration/release.Dockerfile . + docker buildx build --push --platform linux/amd64,linux/arm64 -t olivierkorach/sonar-migration:0.3 -t olivierkorach/sonar-migration:latest -f migration/release.Dockerfile $ROOTDIR fi \ No newline at end of file From 7866d5e0bd1c31f275526d7f41c29d0a9be316c2 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:31:33 +0200 Subject: [PATCH 09/14] Delete only what is needed before rebuilding --- conf/deploy.sh | 2 +- migration/deploy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/deploy.sh b/conf/deploy.sh index 3d4ea21f8..c298bb01d 100755 --- a/conf/deploy.sh +++ b/conf/deploy.sh @@ -48,7 +48,7 @@ while [ $# -ne 0 ]; do done black --line-length=150 . -rm -rf $ROOTDIR/build $ROOTDIR/dist +rm -rf $ROOTDIR/build/lib/sonar $ROOTDIR/build/lib/cli $ROOTDIR/build/scripts*/sonar-tools $ROOTDIR/dist/sonar_tools* python3 $ROOTDIR/setup.py bdist_wheel # Deploy locally for tests diff --git a/migration/deploy.sh b/migration/deploy.sh index a4a56f77e..5f28c2bf0 100755 --- a/migration/deploy.sh +++ b/migration/deploy.sh @@ -44,7 +44,7 @@ while [ $# -ne 0 ]; do done black --line-length=150 . -rm -rf $ROOTDIR/build $ROOTDIR/dist +rm -rf $ROOTDIR/build/lib/migration $ROOTDIR/build/lib/cli $ROOTDIR/build/lib/sonar $ROOTDIR/build/scripts*/sonar_migration $ROOTDIR/dist/sonar_migration* python3 $ROOTDIR/setup_migration.py bdist_wheel # Deploy locally for tests From 7864f76763704ebd6e80656bfceee55e8eccf41a Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:54:02 +0200 Subject: [PATCH 10/14] Fix problem with tests --- sonar/platform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sonar/platform.py b/sonar/platform.py index 3afe69399..280a31b38 100644 --- a/sonar/platform.py +++ b/sonar/platform.py @@ -222,6 +222,7 @@ def __run_request( headers["Authorization"] = f"Bearer {self.__token}" if kwargs.get("with_organization", True): params["organization"] = self.organization + req_type, url = "", "" if log.get_level() >= logging.DEBUG: req_type = getattr(request, "__name__", repr(request)).upper() url = self.__urlstring(api, params) From 8c2d9c416d676502749308f16f4213c700669a3f Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:54:26 +0200 Subject: [PATCH 11/14] Move scanner properties file back in repo root --- conf/scan.sh | 5 ++--- conf/sonar-project.properties => sonar-project.properties | 0 2 files changed, 2 insertions(+), 3 deletions(-) rename conf/sonar-project.properties => sonar-project.properties (100%) diff --git a/conf/scan.sh b/conf/scan.sh index b18ed2951..0d8448725 100755 --- a/conf/scan.sh +++ b/conf/scan.sh @@ -55,17 +55,16 @@ externalIssuesReport="$buildDir/external-issue-report.json" rm -rf -- ${buildDir:?"."}/* .coverage */__pycache__ */*.pyc # mediatools/__pycache__ testpytest/__pycache__ testunittest/__pycache__ if [ "$dotest" == "true" ]; then - ./run_tests.sh + $CONFDIR/run_tests.sh fi if [ "$dolint" != "false" ]; then - ./run_linters.sh + $CONFDIR/run_linters.sh fi version=$(grep PACKAGE_VERSION $ROOTDIR/sonar/version.py | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//") cmd="sonar-scanner -Dsonar.projectVersion=$version \ - -Dproject.settings=$CONFDIR/sonar-project.properties \ -Dsonar.python.flake8.reportPaths=$flake8Report \ -Dsonar.python.pylint.reportPaths=$pylintReport \ -Dsonar.python.bandit.reportPaths=$banditReport \ diff --git a/conf/sonar-project.properties b/sonar-project.properties similarity index 100% rename from conf/sonar-project.properties rename to sonar-project.properties From a309392d08dfd531f72f83bd02d486b8450b2f5d Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:55:55 +0200 Subject: [PATCH 12/14] Fix API docs path in exclusions --- sonar-project.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonar-project.properties b/sonar-project.properties index 94f35155f..b1c9f88b6 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -12,7 +12,7 @@ sonar.python.flake8.reportPaths=build/flake8-report.out sonar.python.pylint.reportPaths=build/pylint-report.out # sonar.python.bandit.reportPaths=build/bandit-report.json -sonar.exclusions=api-doc/**/*, build/**/*, test/**/* -sonar.coverage.exclusions=setup*.py, migration/migration.py, test/**/*, shellcheck2sonar.py, cli/cust_measures.py, sonar/custom_measures.py, cli/support.py, cli/projects_export.py, , cli/projects_import.py +sonar.exclusions=doc/api/**/*, build/**/*, test/**/* +sonar.coverage.exclusions=setup*.py, migration/migration.py, test/**/*, conf/shellcheck2sonar.py, cli/cust_measures.py, sonar/custom_measures.py, cli/support.py, cli/projects_export.py, cli/projects_import.py sonar.cpd.exclusions=setup*.py, migration/migration.py sonar.tests=test From 892324079fc79c96327e0ddeb7c7651414e3efad Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 11:59:43 +0200 Subject: [PATCH 13/14] Don't install bandit, not used anymore --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8aad1b3e7..dc668cdfb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pylint bandit pytest coverage + pip install flake8 pylint pytest coverage if [ -f requirements.txt ]; then pip install -r requirements.txt; fi #- name: Lint with flake8 # run: | From 949a0b94cc1fd5027ca48e7b5834dd17f4cb0708 Mon Sep 17 00:00:00 2001 From: Olivier Korach Date: Sun, 6 Oct 2024 12:00:46 +0200 Subject: [PATCH 14/14] Run linters in conf --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc668cdfb..d4fd0db35 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,8 +49,8 @@ jobs: - name: Run linters working-directory: . run: | - chmod +x ./run_linters.sh - ./run_linters.sh + chmod +x conf/run_linters.sh + conf/run_linters.sh #- name: Cache SonarQube packages # uses: actions/cache@v4 # with: