ci: Start implementing integration tests for xmpp

Author: selfhoster1312

.github/workflows/codeql-analysis.yml

@@ -1,58 +1,84 @@
 name: Development
 on: [push, pull_request]
 jobs:
-  lint:
-    name: golangci-lint
+  # lint:
+  #   name: golangci-lint
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #       with:
+  #         fetch-depth: 20
+  #     - name: Run golangci-lint
+  #       uses: golangci/golangci-lint-action@v3
+  #       with:
+  #         version: latest
+  #         args: "-v --new-from-rev HEAD~5 --timeout=5m"
+  # test-build-upload:
+  #   strategy:
+  #     matrix:
+  #       go-version: [1.22.x]
+  #       platform: [ubuntu-latest]
+  #   runs-on: ${{ matrix.platform }}
+  #   steps:
+  #   - name: Install Go
+  #     uses: actions/setup-go@v3
+  #     with:
+  #       go-version: ${{ matrix.go-version }}
+  #       stable: false
+  #   - name: Checkout code
+  #     uses: actions/checkout@v3
+  #     with:
+  #         fetch-depth: 0
+  #   - name: Test
+  #     run: go test ./... -mod=vendor
+  #   - name: Build
+  #     run: |
+  #       mkdir -p output/{win,lin,arm,mac}
+  #       VERSION=$(git describe --tags)
+  #       CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
+  #       CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
+  #       CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
+  #   - name: Upload linux 64-bit
+  #     if: startsWith(matrix.go-version,'1.22')
+  #     uses: actions/upload-artifact@v3
+  #     with:
+  #       name: matterbridge-linux-64bit
+  #       path: output/lin
+  #   - name: Upload windows 64-bit
+  #     if: startsWith(matrix.go-version,'1.22')
+  #     uses: actions/upload-artifact@v3
+  #     with:
+  #       name: matterbridge-windows-64bit
+  #       path: output/win
+  #   - name: Upload darwin 64-bit
+  #     if: startsWith(matrix.go-version,'1.22')
+  #     uses: actions/upload-artifact@v3
+  #     with:
+  #       name: matterbridge-darwin-64bit
+  #       path: output/mac
+  integration:
+    name: Integration tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
         with:
-          fetch-depth: 20
-      - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3
+          go-version: stable
+      # Install prosody's latest release from upstream
+      - run: sudo wget https://prosody.im/downloads/repos/$(lsb_release -sc)/prosody.sources -O/etc/apt/sources.list.d/prosody.sources
+      - run: sudo apt update
+      - run: sudo apt install -y mercurial lua5.4
+      - run: sudo update-alternatives --set lua-interpreter /usr/bin/lua5.4
+      - run: sudo apt install -y prosody
+      # Setup prosody community modules
+      - run: hg clone https://hg.prosody.im/prosody-modules/ prosody-modules
+      # Copy mod_auth_any to global prosody modules
+      - run: sudo cp -R prosody-modules/mod_auth_any /usr/lib/prosody/modules/
+      # Only one test is run for now
+      - run: ./tests/test.sh xmpp outgoing-message
+      # Upload logs when it failed
+      - uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
         with:
-          version: latest
-          args: "-v --new-from-rev HEAD~5 --timeout=5m"
-  test-build-upload:
-    strategy:
-      matrix:
-        go-version: [1.22.x]
-        platform: [ubuntu-latest]
-    runs-on: ${{ matrix.platform }}
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-        stable: false
-    - name: Checkout code
-      uses: actions/checkout@v3
-      with:
-          fetch-depth: 0
-    - name: Test
-      run: go test ./... -mod=vendor
-    - name: Build
-      run: |
-        mkdir -p output/{win,lin,arm,mac}
-        VERSION=$(git describe --tags)
-        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
-        CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
-        CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
-    - name: Upload linux 64-bit
-      if: startsWith(matrix.go-version,'1.22')
-      uses: actions/upload-artifact@v3
-      with:
-        name: matterbridge-linux-64bit
-        path: output/lin
-    - name: Upload windows 64-bit
-      if: startsWith(matrix.go-version,'1.22')
-      uses: actions/upload-artifact@v3
-      with:
-        name: matterbridge-windows-64bit
-        path: output/win
-    - name: Upload darwin 64-bit
-      if: startsWith(matrix.go-version,'1.22')
-      uses: actions/upload-artifact@v3
-      with:
-        name: matterbridge-darwin-64bit
-        path: output/mac
+          name: logs
+          path: tests/xmpp/*.log

.github/workflows/development.yml

@@ -7,3 +7,5 @@ matterbridge.toml
 
 # Exclude IDE Files
 .vscode
+
+tests/**/*.log

.github/workflows/docker.yml

@@ -62,7 +62,8 @@ func (b *Bxmpp) JoinChannel(channel config.ChannelInfo) error {
 		b.Log.Debugf("using key %s for channel %s", channel.Options.Key, channel.Name)
 		b.xc.JoinProtectedMUC(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"), channel.Options.Key, xmpp.NoHistory, 0, nil)
 	} else {
-		b.xc.JoinMUCNoHistory(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"))
+		// TODO: tihs creates the room if it doesn't exist yet, should it be the default?
+		b.xc.JoinOrCreateMUCNoHistoryDoNotUseOutsideTests(channel.Name+"@"+b.GetString("Muc"), b.GetString("Nick"))
 	}
 	return nil
 }

.gitignore

@@ -0,0 +1,76 @@
+# Integration tests
+
+This directory is for integration tests only. Unit tests reside in their
+respective modules.
+
+## Motivation
+
+Unit testing only checks that assumptions within the code are held. Integration
+tests can provide real-life testing capabilities to ensure that a feature is
+implemented properly, even when going through the remote chat service.
+
+## Methodology
+
+Each matterbrige supported protocol has its own corresponding folder in this
+`tests` folder. Each protocol's integration's tests:
+
+- are run in a specific CI job with a specific environment
+- are documented to be run locally with a specific environment
+- provides implementation for incoming tests, and outgoing tests to avoid
+  feature mismatch (for example, a bridge that could receive attachments
+  from other networks but not the other way around)
+- are run sequentially, although different protocols are tested in parallel
+
+> [!NOTE]
+> `incoming` refers to a message received on that protocol (to be delivered
+> to others via matterbridge), while `outgoing` refers to a message received
+> from another matterbridge protocol to be delivered to this protocol.
+
+## Architecture
+
+At the moment, integration tests require 3 components to play together:
+
+- a real matterbridge instance configured with a protocol account (`pa`) to a protocol room (`pr`)
+- another protocol client (`pc`), connected to the same protocol room (`pr`), following a test scenario
+- a matterbridge API client (`ac`), connected client following the same test scenario (called `ac`)
+
+> [!NOTE]
+> For each tested protocol, the test suite requires two accounts on the remote server.
+
+The `matterbridge.toml` in each integration test folder determines the config
+used for the matterbridge daemon used in the tests.
+
+The API client is located in `tests/api/api.go` and accepts as arguments:
+
+- one of the supported tests (eg. `incoming-message`)
+- a timeout in seconds, after which it will automatically fail (default: 5)
+
+The protocol client is located in `tests/PROTOCOL/PROTOCOL.go`, is a proper
+go module that can be run with `go run .`, and accepts as arguments:
+
+- one of the supported tests (eg. `incoming-message`)
+- a timeout in seconds, after which it will automatically fail (default: 5)
+
+Both binaries:
+
+- run in the background, with the same set of CLI arguments
+- fail the entire pipeline if they individually fail
+- success as soon as the testing criteria is met
+
+For example, in the case of the `incoming-message` test, the protocol client
+will successfully exit as soon as the message is sent without errors,
+but the API client will wait and either:
+
+- successfully exit when receiving the `test-incoming-message`
+- fail after the timeout was reached
+
+## Supported tests
+
+The following test scenarios are supported at the moment:
+
+- `incoming-message` steps:
+  - `pc` sends `test-incoming-message` in `pr` using the native protocol
+  - `ac` receives `<pc> test-incoming-message` in `pr` using the matterbridge API
+- `outgoing-message` steps:
+  - `ac` sends `test-outgoing-message` in `pr` using the matterbridge API
+  - `pc` receives `<ac> test-outgoing-message` in `pr` using the native protocol