diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8aad1b3e7..d4fd0db35 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pylint bandit pytest coverage + pip install flake8 pylint pytest coverage if [ -f requirements.txt ]; then pip install -r requirements.txt; fi #- name: Lint with flake8 # run: | @@ -49,8 +49,8 @@ jobs: - name: Run linters working-directory: . run: | - chmod +x ./run_linters.sh - ./run_linters.sh + chmod +x conf/run_linters.sh + conf/run_linters.sh #- name: Cache SonarQube packages # uses: actions/cache@v4 # with: diff --git a/.sonarcloud.properties b/.sonarcloud.properties deleted file mode 100644 index 248db4465..000000000 --- a/.sonarcloud.properties +++ /dev/null @@ -1,9 +0,0 @@ -# Required metadata -# sonar.projectVersion=1.0 - -# Comma-separated paths to directories with sources (required) -sonar.sources=. -# Encoding of the source files -sonar.sourceEncoding=UTF-8 -sonar.exclusions=**/*.csv,**/*.xls* -sonar.python.version=3.6 \ No newline at end of file diff --git a/.flake8 b/conf/.flake8 similarity index 100% rename from .flake8 rename to conf/.flake8 diff --git a/deploy.bat b/conf/deploy.bat similarity index 100% rename from deploy.bat rename to conf/deploy.bat diff --git a/deploy.sh b/conf/deploy.sh similarity index 72% rename from deploy.sh rename to conf/deploy.sh index 1e23abe1c..c298bb01d 100755 --- a/deploy.sh +++ b/conf/deploy.sh @@ -19,6 +19,10 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + build_docs=1 build_image=1 release=0 @@ -44,19 +48,19 @@ while [ $# -ne 0 ]; do done black --line-length=150 . -rm -rf build dist -python3 setup.py bdist_wheel +rm -rf $ROOTDIR/build/lib/sonar $ROOTDIR/build/lib/cli $ROOTDIR/build/scripts*/sonar-tools $ROOTDIR/dist/sonar_tools* +python3 $ROOTDIR/setup.py bdist_wheel # Deploy locally for tests -pip install --upgrade --force-reinstall dist/sonar_tools-*-py3-*.whl +pip install --upgrade --force-reinstall $ROOTDIR/dist/sonar_tools-*-py3-*.whl if [ "$build_image" == "1" ]; then - docker build -t olivierkorach/sonar-tools:3.5 -t olivierkorach/sonar-tools:latest -f snapshot.Dockerfile . --load + docker build -t olivierkorach/sonar-tools:3.5 -t olivierkorach/sonar-tools:latest -f $CONFDIR/snapshot.Dockerfile $ROOTDIR --load fi if [ "$build_docs" == "1" ]; then - rm -rf api-doc/build - sphinx-build -b html api-doc/source api-doc/build + rm -rf doc/api/build + sphinx-build -b html doc/api/source doc/api/build fi # Deploy on pypi.org once released @@ -64,10 +68,10 @@ if [ "$release" = "1" ]; then echo "Confirm release [y/n] ?" read -r confirm if [ "$confirm" = "y" ]; then - python3 -m twine upload dist/sonar_tools-*-py3-*.whl + python3 -m twine upload $ROOTDIR/dist/sonar_tools-*-py3-*.whl fi fi if [ "$release_docker" = "1" ]; then - docker buildx build --push --platform linux/amd64,linux/arm64 -t olivierkorach/sonar-tools:3.5 -t olivierkorach/sonar-tools:latest -f release.Dockerfile . + docker buildx build --push --platform linux/amd64,linux/arm64 -t olivierkorach/sonar-tools:3.5 -t olivierkorach/sonar-tools:latest -f $ROOTDIR/release.Dockerfile . fi \ No newline at end of file diff --git a/pylintrc b/conf/pylintrc similarity index 100% rename from pylintrc rename to conf/pylintrc diff --git a/release.Dockerfile b/conf/release.Dockerfile similarity index 100% rename from release.Dockerfile rename to conf/release.Dockerfile diff --git a/run_linters.sh b/conf/run_linters.sh similarity index 75% rename from run_linters.sh rename to conf/run_linters.sh index 306ab1009..e7300d911 100755 --- a/run_linters.sh +++ b/conf/run_linters.sh @@ -19,7 +19,11 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # -buildDir="build" +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +buildDir="$ROOTDIR/build" pylintReport="$buildDir/pylint-report.out" # banditReport="$buildDir/bandit-report.json" flake8Report="$buildDir/flake8-report.out" @@ -29,7 +33,7 @@ externalIssueReport="$buildDir/external-issue-report.json" echo "Running pylint" rm -f $pylintReport -pylint ./*.py ./*/*.py -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" | tee $pylintReport +pylint --rcfile $CONFDIR/pylintrc $ROOTDIR/*.py $ROOTDIR/*/*.py -r n --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" | tee $pylintReport re=$? if [ "$re" == "32" ]; then >&2 echo "ERROR: pylint execution failed, errcode $re, aborting..." @@ -39,11 +43,11 @@ fi echo "Running flake8" rm -f $flake8Report # See .flake8 file for settings -flake8 . >$flake8Report +flake8 --config "$CONFIG/.flake8" "$ROOTDIR" >$flake8Report # echo "Running bandit" # rm -f $banditReport # bandit --exit-zero -f json --skip B311,B303,B101 -r . -x .vscode,./tests >$banditReport echo "Running shellcheck" -shellcheck ./*.sh ./*/*.sh -s bash -f json | ./shellcheck2sonar.py >$externalIssueReport +shellcheck $ROOTDIR/*.sh $ROOTDIR/*/*.sh -s bash -f json | $CONFDIR/shellcheck2sonar.py >$externalIssueReport diff --git a/run_tests.sh b/conf/run_tests.sh similarity index 74% rename from run_tests.sh rename to conf/run_tests.sh index 2efe1bec9..ce7931e65 100755 --- a/run_tests.sh +++ b/conf/run_tests.sh @@ -19,14 +19,18 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # -buildDir="build" +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +buildDir="$ROOTDIR/build" coverageReport="$buildDir/coverage.xml" [ ! -d $buildDir ] && mkdir $buildDir echo "Running tests" -chmod u-rw test/sif_not_readable.json +unreadableSif=$ROOTDIR/test/sif_not_readable.json +chmod u-rw $unreadableSif export SONAR_HOST_URL=${1:-${SONAR_HOST_URL_TEST}} -coverage run --source=. -m pytest test/ +coverage run --source=$ROOTDIR -m pytest $ROOTDIR/test/ coverage xml -o $coverageReport -chmod u+rw test/sif_not_readable.json +chmod u+rw $unreadableSif diff --git a/scan.sh b/conf/scan.sh similarity index 84% rename from scan.sh rename to conf/scan.sh index 0ac37e681..0d8448725 100755 --- a/scan.sh +++ b/conf/scan.sh @@ -19,6 +19,10 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + dolint=true dotest=false @@ -40,7 +44,7 @@ do shift done -buildDir="build" +buildDir="$ROOTDIR/build" pylintReport="$buildDir/pylint-report.out" banditReport="$buildDir/bandit-report.json" flake8Report="$buildDir/flake8-report.out" @@ -51,14 +55,14 @@ externalIssuesReport="$buildDir/external-issue-report.json" rm -rf -- ${buildDir:?"."}/* .coverage */__pycache__ */*.pyc # mediatools/__pycache__ testpytest/__pycache__ testunittest/__pycache__ if [ "$dotest" == "true" ]; then - ./run_tests.sh + $CONFDIR/run_tests.sh fi if [ "$dolint" != "false" ]; then - ./run_linters.sh + $CONFDIR/run_linters.sh fi -version=$(grep PACKAGE_VERSION sonar/version.py | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//") +version=$(grep PACKAGE_VERSION $ROOTDIR/sonar/version.py | cut -d "=" -f 2 | sed -e "s/[\'\" ]//g" -e "s/^ +//" -e "s/ +$//") cmd="sonar-scanner -Dsonar.projectVersion=$version \ -Dsonar.python.flake8.reportPaths=$flake8Report \ diff --git a/shellcheck2sonar.py b/conf/shellcheck2sonar.py similarity index 100% rename from shellcheck2sonar.py rename to conf/shellcheck2sonar.py diff --git a/snapshot.Dockerfile b/conf/snapshot.Dockerfile similarity index 100% rename from snapshot.Dockerfile rename to conf/snapshot.Dockerfile diff --git a/api-doc/Makefile b/doc/api/Makefile similarity index 100% rename from api-doc/Makefile rename to doc/api/Makefile diff --git a/api-doc/make.bat b/doc/api/make.bat similarity index 100% rename from api-doc/make.bat rename to doc/api/make.bat diff --git a/api-doc/source/aggregation_permissions.rst b/doc/api/source/aggregation_permissions.rst similarity index 100% rename from api-doc/source/aggregation_permissions.rst rename to doc/api/source/aggregation_permissions.rst diff --git a/api-doc/source/aggregations.rst b/doc/api/source/aggregations.rst similarity index 100% rename from api-doc/source/aggregations.rst rename to doc/api/source/aggregations.rst diff --git a/api-doc/source/application_permissions.rst b/doc/api/source/application_permissions.rst similarity index 100% rename from api-doc/source/application_permissions.rst rename to doc/api/source/application_permissions.rst diff --git a/api-doc/source/applications.rst b/doc/api/source/applications.rst similarity index 100% rename from api-doc/source/applications.rst rename to doc/api/source/applications.rst diff --git a/api-doc/source/branches.rst b/doc/api/source/branches.rst similarity index 100% rename from api-doc/source/branches.rst rename to doc/api/source/branches.rst diff --git a/api-doc/source/components.rst b/doc/api/source/components.rst similarity index 100% rename from api-doc/source/components.rst rename to doc/api/source/components.rst diff --git a/api-doc/source/conf.py b/doc/api/source/conf.py similarity index 100% rename from api-doc/source/conf.py rename to doc/api/source/conf.py diff --git a/api-doc/source/devops.rst b/doc/api/source/devops.rst similarity index 100% rename from api-doc/source/devops.rst rename to doc/api/source/devops.rst diff --git a/api-doc/source/findings.rst b/doc/api/source/findings.rst similarity index 100% rename from api-doc/source/findings.rst rename to doc/api/source/findings.rst diff --git a/api-doc/source/global_permissions.rst b/doc/api/source/global_permissions.rst similarity index 100% rename from api-doc/source/global_permissions.rst rename to doc/api/source/global_permissions.rst diff --git a/api-doc/source/groups.rst b/doc/api/source/groups.rst similarity index 100% rename from api-doc/source/groups.rst rename to doc/api/source/groups.rst diff --git a/api-doc/source/hotspots.rst b/doc/api/source/hotspots.rst similarity index 100% rename from api-doc/source/hotspots.rst rename to doc/api/source/hotspots.rst diff --git a/api-doc/source/index.rst b/doc/api/source/index.rst similarity index 100% rename from api-doc/source/index.rst rename to doc/api/source/index.rst diff --git a/api-doc/source/issues.rst b/doc/api/source/issues.rst similarity index 100% rename from api-doc/source/issues.rst rename to doc/api/source/issues.rst diff --git a/api-doc/source/languages.rst b/doc/api/source/languages.rst similarity index 100% rename from api-doc/source/languages.rst rename to doc/api/source/languages.rst diff --git a/api-doc/source/measures.rst b/doc/api/source/measures.rst similarity index 100% rename from api-doc/source/measures.rst rename to doc/api/source/measures.rst diff --git a/api-doc/source/metrics.rst b/doc/api/source/metrics.rst similarity index 100% rename from api-doc/source/metrics.rst rename to doc/api/source/metrics.rst diff --git a/api-doc/source/organizations.rst b/doc/api/source/organizations.rst similarity index 100% rename from api-doc/source/organizations.rst rename to doc/api/source/organizations.rst diff --git a/api-doc/source/permission_templates.rst b/doc/api/source/permission_templates.rst similarity index 100% rename from api-doc/source/permission_templates.rst rename to doc/api/source/permission_templates.rst diff --git a/api-doc/source/permissions.rst b/doc/api/source/permissions.rst similarity index 100% rename from api-doc/source/permissions.rst rename to doc/api/source/permissions.rst diff --git a/api-doc/source/platform.rst b/doc/api/source/platform.rst similarity index 100% rename from api-doc/source/platform.rst rename to doc/api/source/platform.rst diff --git a/api-doc/source/portfolio_permissions.rst b/doc/api/source/portfolio_permissions.rst similarity index 100% rename from api-doc/source/portfolio_permissions.rst rename to doc/api/source/portfolio_permissions.rst diff --git a/api-doc/source/portfolios.rst b/doc/api/source/portfolios.rst similarity index 100% rename from api-doc/source/portfolios.rst rename to doc/api/source/portfolios.rst diff --git a/api-doc/source/project_permissions.rst b/doc/api/source/project_permissions.rst similarity index 100% rename from api-doc/source/project_permissions.rst rename to doc/api/source/project_permissions.rst diff --git a/api-doc/source/projects.rst b/doc/api/source/projects.rst similarity index 100% rename from api-doc/source/projects.rst rename to doc/api/source/projects.rst diff --git a/api-doc/source/pullrequests.rst b/doc/api/source/pullrequests.rst similarity index 100% rename from api-doc/source/pullrequests.rst rename to doc/api/source/pullrequests.rst diff --git a/api-doc/source/quality_permissions.rst b/doc/api/source/quality_permissions.rst similarity index 100% rename from api-doc/source/quality_permissions.rst rename to doc/api/source/quality_permissions.rst diff --git a/api-doc/source/qualitygate_permissions.rst b/doc/api/source/qualitygate_permissions.rst similarity index 100% rename from api-doc/source/qualitygate_permissions.rst rename to doc/api/source/qualitygate_permissions.rst diff --git a/api-doc/source/qualitygates.rst b/doc/api/source/qualitygates.rst similarity index 100% rename from api-doc/source/qualitygates.rst rename to doc/api/source/qualitygates.rst diff --git a/api-doc/source/qualityprofile_permissions.rst b/doc/api/source/qualityprofile_permissions.rst similarity index 100% rename from api-doc/source/qualityprofile_permissions.rst rename to doc/api/source/qualityprofile_permissions.rst diff --git a/api-doc/source/qualityprofiles.rst b/doc/api/source/qualityprofiles.rst similarity index 100% rename from api-doc/source/qualityprofiles.rst rename to doc/api/source/qualityprofiles.rst diff --git a/api-doc/source/rules.rst b/doc/api/source/rules.rst similarity index 100% rename from api-doc/source/rules.rst rename to doc/api/source/rules.rst diff --git a/api-doc/source/sq_object.rst b/doc/api/source/sq_object.rst similarity index 100% rename from api-doc/source/sq_object.rst rename to doc/api/source/sq_object.rst diff --git a/api-doc/source/tasks.rst b/doc/api/source/tasks.rst similarity index 100% rename from api-doc/source/tasks.rst rename to doc/api/source/tasks.rst diff --git a/api-doc/source/tokens.rst b/doc/api/source/tokens.rst similarity index 100% rename from api-doc/source/tokens.rst rename to doc/api/source/tokens.rst diff --git a/api-doc/source/users.rst b/doc/api/source/users.rst similarity index 100% rename from api-doc/source/users.rst rename to doc/api/source/users.rst diff --git a/api-doc/source/webhooks.rst b/doc/api/source/webhooks.rst similarity index 100% rename from api-doc/source/webhooks.rst rename to doc/api/source/webhooks.rst diff --git a/migration.Dockerfile b/migration.Dockerfile deleted file mode 100644 index 35cb679c8..000000000 --- a/migration.Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -FROM alpine:3.20.3 -LABEL maintainer="olivier.korach@gmail.com" -ENV IN_DOCKER="Yes" - -ARG USERNAME=sonar -ARG USER_UID=1000 -ARG GROUPNAME=sonar - -# Create the user -RUN addgroup -S ${GROUPNAME} && adduser -u ${USER_UID} -S ${USERNAME} -G ${GROUPNAME} - -# Install python/pip -ENV PYTHONUNBUFFERED=1 -RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python - -# create a virtual environment and add it to PATH so that it is -# applied for all future RUN and CMD calls -ENV VIRTUAL_ENV=/opt/venv -RUN python3 -m venv ${VIRTUAL_ENV} -ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" - -WORKDIR /opt/sonar-migration - -COPY ./sonar sonar -COPY ./requirements.txt . -COPY ./cli cli -COPY ./setup_migration.py . -COPY ./sonar_migration . -COPY ./README.md . -COPY ./LICENSE . -COPY ./sonar/audit sonar/audit - -RUN pip install --upgrade pip \ -&& pip install --no-cache-dir -r requirements.txt \ -&& pip install --no-cache-dir --upgrade pip setuptools wheel \ -&& python setup_migration.py bdist_wheel \ -&& pip install dist/sonar_migration-py3-*.whl --force-reinstall - -USER ${USERNAME} -WORKDIR /home/${USERNAME} - -ENTRYPOINT ["sonar-migration"] diff --git a/migration/deploy.sh b/migration/deploy.sh index 8f82a635c..5f28c2bf0 100755 --- a/migration/deploy.sh +++ b/migration/deploy.sh @@ -19,6 +19,10 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # +ME="$( basename "${BASH_SOURCE[0]}" )" +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" +CONFDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + build_image=1 release=0 @@ -40,14 +44,14 @@ while [ $# -ne 0 ]; do done black --line-length=150 . -rm -rf build dist -python3 setup_migration.py bdist_wheel +rm -rf $ROOTDIR/build/lib/migration $ROOTDIR/build/lib/cli $ROOTDIR/build/lib/sonar $ROOTDIR/build/scripts*/sonar_migration $ROOTDIR/dist/sonar_migration* +python3 $ROOTDIR/setup_migration.py bdist_wheel # Deploy locally for tests -pip install --upgrade --force-reinstall dist/sonar_migration-*-py3-*.whl +pip install --upgrade --force-reinstall $ROOTDIR/dist/sonar_migration-*-py3-*.whl if [ "$build_image" == "1" ]; then - docker build -t olivierkorach/sonar-migration:latest -f migration/snapshot.Dockerfile . --load + docker build -t olivierkorach/sonar-migration:latest -f migration/snapshot.Dockerfile $ROOTDIR --load fi # Deploy on pypi.org once released @@ -55,10 +59,10 @@ if [ "$release" = "1" ]; then echo "Confirm release [y/n] ?" read -r confirm if [ "$confirm" = "y" ]; then - python3 -m twine upload dist/sonar_migration-*-py3-*.whl + python3 -m twine upload $ROOTDIR/dist/sonar_migration-*-py3-*.whl fi fi if [ "$release_docker" = "1" ]; then - docker buildx build --push --platform linux/amd64,linux/arm64 -t olivierkorach/sonar-migration:0.3 -t olivierkorach/sonar-migration:latest -f migration/release.Dockerfile . + docker buildx build --push --platform linux/amd64,linux/arm64 -t olivierkorach/sonar-migration:0.3 -t olivierkorach/sonar-migration:latest -f migration/release.Dockerfile $ROOTDIR fi \ No newline at end of file diff --git a/sonar-project.properties b/sonar-project.properties index 94f35155f..b1c9f88b6 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -12,7 +12,7 @@ sonar.python.flake8.reportPaths=build/flake8-report.out sonar.python.pylint.reportPaths=build/pylint-report.out # sonar.python.bandit.reportPaths=build/bandit-report.json -sonar.exclusions=api-doc/**/*, build/**/*, test/**/* -sonar.coverage.exclusions=setup*.py, migration/migration.py, test/**/*, shellcheck2sonar.py, cli/cust_measures.py, sonar/custom_measures.py, cli/support.py, cli/projects_export.py, , cli/projects_import.py +sonar.exclusions=doc/api/**/*, build/**/*, test/**/* +sonar.coverage.exclusions=setup*.py, migration/migration.py, test/**/*, conf/shellcheck2sonar.py, cli/cust_measures.py, sonar/custom_measures.py, cli/support.py, cli/projects_export.py, cli/projects_import.py sonar.cpd.exclusions=setup*.py, migration/migration.py sonar.tests=test diff --git a/sonar/platform.py b/sonar/platform.py index 3afe69399..280a31b38 100644 --- a/sonar/platform.py +++ b/sonar/platform.py @@ -222,6 +222,7 @@ def __run_request( headers["Authorization"] = f"Bearer {self.__token}" if kwargs.get("with_organization", True): params["organization"] = self.organization + req_type, url = "", "" if log.get_level() >= logging.DEBUG: req_type = getattr(request, "__name__", repr(request)).upper() url = self.__urlstring(api, params)