diff --git a/tests/acceptance/TestHelpers/OcisConfigHelper.php b/tests/acceptance/TestHelpers/OcisConfigHelper.php index 5d3ac479af3..e45340b7837 100644 --- a/tests/acceptance/TestHelpers/OcisConfigHelper.php +++ b/tests/acceptance/TestHelpers/OcisConfigHelper.php @@ -120,4 +120,20 @@ public static function startOcis(): ResponseInterface { $url = self::getWrapperUrl() . "/start"; return self::sendRequest($url, "POST"); } + + /** + * this method stops the running oCIS instance, + * restarts oCIS without specific services, + * and then starts the excluded services separately. + * + * @param string $service + * @param array $envs + * + * @return ResponseInterface + * @throws GuzzleException + */ + public static function startService(string $service, array $envs = []): ResponseInterface { + $url = self::getWrapperUrl() . "/services/" . $service; + return self::sendRequest($url, "POST", \json_encode($envs)); + } } diff --git a/tests/acceptance/bootstrap/OcisConfigContext.php b/tests/acceptance/bootstrap/OcisConfigContext.php index 286a9acbfdc..cee4163d41a 100644 --- a/tests/acceptance/bootstrap/OcisConfigContext.php +++ b/tests/acceptance/bootstrap/OcisConfigContext.php @@ -191,6 +191,32 @@ public function theConfigHasBeenSetToValue(TableNode $table): void { ); } + /** + * @Given the administrator has started service :service separately with the following configs: + * + * @param string $service + * @param TableNode $table + * + * @return void + * @throws GuzzleException + */ + public function theAdministratorHasStartedServiceSeparatelyWithTheFollowingConfig( + string $service, + TableNode $table + ): void { + $envs = []; + foreach ($table->getHash() as $row) { + $envs[$row['config']] = $row['value']; + } + + $response = OcisConfigHelper::startService($service, $envs); + Assert::assertEquals( + 200, + $response->getStatusCode(), + "Failed to start service $service." + ); + } + /** * @AfterScenario @env-config * diff --git a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md index b39621ca7bf..424b8a41793 100644 --- a/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-localAPI-on-OCIS-storage.md @@ -321,8 +321,9 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [apiSharingNg1/propfindShares.feature:149](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiSharingNg1/propfindShares.feature#L149) #### [Readiness check for some services returns 500 status code](https://github.com/owncloud/ocis/issues/10661) -- [apiServiceAvailability/serviceAvailabilityCheck.feature:116](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature#L116) -- [apiServiceAvailability/serviceAvailabilityCheck.feature:125](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature#L125) +- [apiServiceAvailability/serviceAvailabilityCheck.feature:111](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature#L111) +- [apiServiceAvailability/serviceAvailabilityCheck.feature:120](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature#L120) +- [apiServiceAvailability/serviceAvailabilityCheck.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature#L131) Note: always have an empty line at the end of this file. The bash script that processes this file requires that the last line has a newline on the end. diff --git a/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature b/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature index 6fffb3e8104..c927d9691a7 100644 --- a/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature +++ b/tests/acceptance/features/apiServiceAvailability/serviceAvailabilityCheck.feature @@ -43,15 +43,13 @@ Feature: service health check @env-config Scenario: check extra services health Given the following configs have been set: - | config | value | - | OCIS_ADD_RUN_SERVICES | audit,auth-app,auth-bearer,policies,invitations | - | AUDIT_DEBUG_ADDR | 0.0.0.0:9229 | - | AUTH_APP_DEBUG_ADDR | 0.0.0.0:9245 | - | POLICIES_DEBUG_ADDR | 0.0.0.0:9129 | - | INVITATIONS_DEBUG_ADDR | 0.0.0.0:9269 | + | config | value | + | OCIS_ADD_RUN_SERVICES | auth-app,policies,invitations | + | AUTH_APP_DEBUG_ADDR | 0.0.0.0:9245 | + | POLICIES_DEBUG_ADDR | 0.0.0.0:9129 | + | INVITATIONS_DEBUG_ADDR | 0.0.0.0:9269 | When a user requests these URLs with "GET" and no authentication | endpoint | service | - | http://%base_url_hostname%:9229/healthz | audit | | http://%base_url_hostname%:9245/healthz | auth-app | | http://%base_url_hostname%:9269/healthz | invitations | | http://%base_url_hostname%:9129/healthz | policies | @@ -97,23 +95,20 @@ Feature: service health check @env-config Scenario: check extra services readiness Given the following configs have been set: - | config | value | - | OCIS_ADD_RUN_SERVICES | audit,auth-app,auth-bearer,policies,invitations | - | AUDIT_DEBUG_ADDR | 0.0.0.0:9229 | - | AUTH_APP_DEBUG_ADDR | 0.0.0.0:9245 | - | AUTH_BEARER_DEBUG_ADDR | 0.0.0.0:9149 | - | POLICIES_DEBUG_ADDR | 0.0.0.0:9129 | - | INVITATIONS_DEBUG_ADDR | 0.0.0.0:9269 | + | config | value | + | OCIS_ADD_RUN_SERVICES | auth-app,policies,invitations | + | AUTH_APP_DEBUG_ADDR | 0.0.0.0:9245 | + | POLICIES_DEBUG_ADDR | 0.0.0.0:9129 | + | INVITATIONS_DEBUG_ADDR | 0.0.0.0:9269 | When a user requests these URLs with "GET" and no authentication | endpoint | service | - | http://%base_url_hostname%:9229/readyz | audit | | http://%base_url_hostname%:9245/readyz | auth-app | | http://%base_url_hostname%:9269/readyz | invitations | | http://%base_url_hostname%:9129/readyz | policies | Then the HTTP status code of responses on all endpoints should be "200" @issue-10661 - Scenario: check default services readiness + Scenario: check default services readiness (graph, idp, proxy) When a user requests these URLs with "GET" and no authentication | endpoint | service | | http://%base_url_hostname%:9124/readyz | graph | @@ -122,7 +117,18 @@ Feature: service health check Then the HTTP status code of responses on all endpoints should be "200" @env-config @issue-10661 - Scenario: check extra services readiness + Scenario: check auth-bearer service health + Given the following configs have been set: + | config | value | + | OCIS_ADD_RUN_SERVICES | auth-bearer | + | AUTH_BEARER_DEBUG_ADDR | 0.0.0.0:9149 | + When a user requests these URLs with "GET" and no authentication + | endpoint | service | + | http://%base_url_hostname%:9149/healthz | auth-bearer | + Then the HTTP status code should be "200" + + @env-config @issue-10661 + Scenario: check auth-bearer service readiness Given the following configs have been set: | config | value | | OCIS_ADD_RUN_SERVICES | auth-bearer | @@ -130,4 +136,16 @@ Feature: service health check When a user requests these URLs with "GET" and no authentication | endpoint | service | | http://%base_url_hostname%:9149/readyz | auth-bearer | + Then the HTTP status code should be "200" + + @env-config + Scenario: check services health and readiness while running separately + Given the administrator has started service "audit" separately with the following configs: + | config | value | + | OCIS_LOG_LEVEL | info | + | AUDIT_DEBUG_ADDR | 0.0.0.0:9229 | + When a user requests these URLs with "GET" and no authentication + | endpoint | service | + | http://%base_url_hostname%:9229/healthz | audit | + | http://%base_url_hostname%:9229/readyz | audit | Then the HTTP status code of responses on all endpoints should be "200" diff --git a/tests/ociswrapper/README.md b/tests/ociswrapper/README.md index 6186e272678..fbacdedba60 100644 --- a/tests/ociswrapper/README.md +++ b/tests/ociswrapper/README.md @@ -124,3 +124,39 @@ Also, see `./bin/ociswrapper help` for more information. - `200 OK` - oCIS server is stopped - `500 Internal Server Error` - Unable to stop oCIS server + +6. `POST /services/{service-name}` + + Restart oCIS instances without specified service and start that service independently (not covered by the oCIS supervisor). + + Body of the request should be a JSON object with the following structure: + + ```json + { + "ENV_KEY1": "value1", + "ENV_KEY2": "value2" + } + ``` + + > **⚠️ Note:** + > + > You need to set the proper addresses to access the service from other steps in the CI pipeline. + > + > `{SERVICE-NAME}_DEBUG_ADDR=0.0.0.0:{DEBUG_PORT}` + > + > `{SERVICE-NAME}_HTTP_ADDR=0.0.0.0:{HTTP_PORT}` + + Returns: + + - `200 OK` - oCIS service started successfully + - `400 Bad Request` - request body is not a valid JSON object + - `500 Internal Server Error` - Failed to start oCIS service audit + +7. `DELETE /services/{service-name}` + + Stop individually running oCIS service + + Returns: + + - `200 OK` - oCIS service stopped successfully + - `500 Internal Server Error` - Unable to stop oCIS service diff --git a/tests/ociswrapper/ocis/config/config.go b/tests/ociswrapper/ocis/config/config.go index 904d42b21e8..600f570ad87 100644 --- a/tests/ociswrapper/ocis/config/config.go +++ b/tests/ociswrapper/ocis/config/config.go @@ -8,6 +8,49 @@ var config = map[string]string{ "adminPassword": "", } +var services = map[string]int{ + "ocis": 9250, + "activitylog": 9197, + "app-provider": 9165, + "app-registry": 9243, + "audit": 9149, + "auth-app": 9245, + "auth-bearer": 9149, + "auth-basic": 9147, + "auth-machine": 9167, + "auth-service": 9198, + "clientlog": 9260, + "eventhistory": 9270, + "frontend": 9141, + "gateway": 9143, + "graph": 9124, + "groups": 9161, + "idm": 9239, + "idp": 9134, + "invitations": 9269, + "nats": 9234, + "ocdav": 9163, + "ocm": 9281, + "ocs": 9114, + "policies": 9129, + "postprocessing": 9255, + "proxy": 9205, + "search": 9224, + "settings": 9194, + "sharing": 9151, + "sse": 9139, + "storage-publiclink": 9179, + "storage-shares": 9156, + "storage-system": 9217, + "storage-users": 9159, + "thumbnails": 9189, + "userlog": 9214, + "users": 9145, + "web": 9104, + "webdav": 9119, + "webfinger": 9279, +} + func Set(key string, value string) { config[key] = value } @@ -15,3 +58,11 @@ func Set(key string, value string) { func Get(key string) string { return config[key] } + +func SetServiceDebugPort(key string, value int) { + services[key] = value +} + +func GetServiceDebugPort(key string) int { + return services[key] +} diff --git a/tests/ociswrapper/ocis/ocis.go b/tests/ociswrapper/ocis/ocis.go index aa4a69dd997..c9f44c0bfc1 100644 --- a/tests/ociswrapper/ocis/ocis.go +++ b/tests/ociswrapper/ocis/ocis.go @@ -6,6 +6,7 @@ import ( "crypto/tls" "fmt" "io" + "net" "net/http" "os" "os/exec" @@ -26,110 +27,31 @@ var cmd *exec.Cmd var retryCount = 0 var stopSignal = false var EnvConfigs = []string{} +var runningServices = make(map[string]int) func Start(envMap []string) { - // wait for the log scanner to finish - var wg sync.WaitGroup - wg.Add(2) - - stopSignal = false - if retryCount == 0 { - defer common.Wg.Done() - } - - cmd = exec.Command(config.Get("bin"), "server") - if envMap == nil { - cmd.Env = append(os.Environ(), EnvConfigs...) - } else { - cmd.Env = append(os.Environ(), envMap...) - } - - logs, err := cmd.StderrPipe() - if err != nil { - log.Panic(err) - } - output, err := cmd.StdoutPipe() - if err != nil { - log.Panic(err) - } - - err = cmd.Start() - if err != nil { - log.Panic(err) - } - - logScanner := bufio.NewScanner(logs) - outputScanner := bufio.NewScanner(output) - outChan := make(chan string) - - // Read the logs when the 'ocis server' command is running - go func() { - defer wg.Done() - for logScanner.Scan() { - outChan <- logScanner.Text() - } - }() - - go func() { - defer wg.Done() - for outputScanner.Scan() { - outChan <- outputScanner.Text() - } - }() - - // Fetch logs from the channel and print them - go func() { - for s := range outChan { - fmt.Println(s) - } - }() - - if err := cmd.Wait(); err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - status := exitErr.Sys().(syscall.WaitStatus) - // retry only if oCIS server exited with code > 0 - // -1 exit code means that the process was killed by a signal (syscall.SIGINT) - if status.ExitStatus() > 0 && !stopSignal { - waitUntilCompleteShutdown() - - log.Println(fmt.Sprintf("oCIS server exited with code %v", status.ExitStatus())) - - // retry to start oCIS server - retryCount++ - maxRetry, _ := strconv.Atoi(config.Get("retry")) - if retryCount <= maxRetry { - wg.Wait() - close(outChan) - log.Println(fmt.Sprintf("Retry starting oCIS server... (retry %v)", retryCount)) - // wait 500 milliseconds before retrying - time.Sleep(500 * time.Millisecond) - Start(envMap) - return - } - } - } - } - wg.Wait() - close(outChan) + log.Println("Starting oCIS service...") + StartService("", envMap) } func Stop() (bool, string) { log.Println("Stopping oCIS server...") stopSignal = true - if cmd == nil { - return true, "oCIS server is not running" + var stopErrors []string + for service := range runningServices { + success, message := StopService(service) + if !success { + stopErrors = append(stopErrors, message) + } } - - err := cmd.Process.Signal(syscall.SIGINT) - if err != nil { - if !strings.HasSuffix(err.Error(), "process already finished") { - log.Fatalln(err) - } else { - return true, "oCIS server is already stopped" + if len(stopErrors) > 0 { + log.Println("Errors occurred while stopping services:") + for _, err := range stopErrors { + log.Println(err) } } - cmd.Process.Wait() + success, message := waitUntilCompleteShutdown() cmd = nil @@ -147,10 +69,16 @@ func Restart(envMap []string) (bool, string) { } func IsOcisRunning() bool { - if cmd != nil { - return cmd.Process.Pid > 0 + if runningServices["ocis"] == 0 { + return false } - return false + + _, err := os.FindProcess(runningServices["ocis"]) + if err != nil { + delete(runningServices, "ocis") + return false + } + return true } func waitAllServices(startTime time.Time, timeout time.Duration) { @@ -271,3 +199,177 @@ func RunCommand(command string, inputs []string) (int, string) { return c.ProcessState.ExitCode(), cmdOutput } + +func StartService(service string, envMap []string) { + // Initialize command args based on service presence + cmdArgs := []string{"server"} // Default command args + + if service != "" { + cmdArgs = append([]string{service}, cmdArgs...) + } + // wait for the log scanner to finish + var wg sync.WaitGroup + wg.Add(2) + + stopSignal = false + if retryCount == 0 { + defer common.Wg.Done() + } + + cmd = exec.Command(config.Get("bin"), cmdArgs...) + + if len(envMap) == 0 { + cmd.Env = append(os.Environ(), EnvConfigs...) + } else { + cmd.Env = append(os.Environ(), envMap...) + } + + logs, err := cmd.StderrPipe() + if err != nil { + log.Panic(err) + } + output, err := cmd.StdoutPipe() + if err != nil { + log.Panic(err) + } + + err = cmd.Start() + + if err != nil { + log.Panic(err) + } + + logScanner := bufio.NewScanner(logs) + outputScanner := bufio.NewScanner(output) + outChan := make(chan string) + + if service == "" { + runningServices["ocis"] = cmd.Process.Pid + } else { + runningServices[service] = cmd.Process.Pid + } + + for listService, pid := range runningServices { + log.Println(fmt.Sprintf("%s service started with process id %v", listService, pid)) + } + + // Read the logs when the 'ocis server' command is running + go func() { + defer wg.Done() + for logScanner.Scan() { + outChan <- logScanner.Text() + } + }() + + go func() { + defer wg.Done() + for outputScanner.Scan() { + outChan <- outputScanner.Text() + } + }() + + // Fetch logs from the channel and print them + go func() { + for s := range outChan { + fmt.Println(s) + } + }() + + if err := cmd.Wait(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + status := exitErr.Sys().(syscall.WaitStatus) + // retry only if oCIS server exited with code > 0 + // -1 exit code means that the process was killed by a signal (syscall.SIGINT) + if status.ExitStatus() > 0 && !stopSignal { + waitUntilCompleteShutdown() + + log.Println(fmt.Sprintf("oCIS server exited with code %v", status.ExitStatus())) + + // retry to start oCIS server + retryCount++ + maxRetry, _ := strconv.Atoi(config.Get("retry")) + if retryCount <= maxRetry { + wg.Wait() + close(outChan) + log.Println(fmt.Sprintf("Retry starting oCIS server... (retry %v)", retryCount)) + // wait 500 milliseconds before retrying + time.Sleep(500 * time.Millisecond) + StartService(service, envMap) + return + } + } + } + } + wg.Wait() + close(outChan) +} + +// Stop oCIS service or a specific service by its unique identifier +func StopService(service string) (bool, string) { + pid, exists := runningServices[service] + if !exists { + return false, fmt.Sprintf("Service %s is not running", service) + } + + process, err := os.FindProcess(pid) + if err != nil { + return false, fmt.Sprintf("Failed to find service %s process running with ID %d", service, pid) + } + + pKillError := process.Signal(syscall.SIGINT) + if pKillError != nil { + return false, fmt.Sprintf("Failed to stop service with process id %d", pid) + } + + success := WaitForServiceStatus(service, false) + if !success { + StopService(service) + } + + delete(runningServices, service) + + return true, fmt.Sprintf("Service %s stopped successfully", service) +} + +func WaitForServiceStatus(service string, waitForUp bool) bool { + overallTimeout := time.After(30 * time.Second) + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + port := config.GetServiceDebugPort(service) + + for { + select { + case <-overallTimeout: + status := "available" + if !waitForUp { + status = "shut down" + } + log.Println(fmt.Errorf("Timeout: %s service did not %s within 30 seconds", service, status).Error()) + return false + case <-ticker.C: + if _, exists := runningServices[service]; !exists { + log.Println(fmt.Sprintf("Service %s not found in running services. Retrying...", service)) + continue + } + + address := fmt.Sprintf(":%d", port) + conn, err := net.DialTimeout("tcp", address, 1*time.Second) + if waitForUp { + if err == nil { + _ = conn.Close() + log.Println(fmt.Sprintf("%s service is ready to listen on port %d", service, port)) + return true + } + log.Println(fmt.Sprintf("%s service is not ready on port %d. %v", service, port, err)) + } else { + if err != nil { + log.Println(fmt.Sprintf("%s service port %d is no longer reachable", service, port)) + return true + } + _ = conn.Close() + log.Println(fmt.Sprintf("%s service port %d is still active. Retrying...", service, port)) + } + } + } +} diff --git a/tests/ociswrapper/wrapper/handlers/handler.go b/tests/ociswrapper/wrapper/handlers/handler.go index 16ef3158c80..6d88be21f58 100644 --- a/tests/ociswrapper/wrapper/handlers/handler.go +++ b/tests/ociswrapper/wrapper/handlers/handler.go @@ -5,9 +5,13 @@ import ( "errors" "fmt" "io" + "log" "net/http" + "strings" + "strconv" "ociswrapper/common" "ociswrapper/ocis" + "ociswrapper/ocis/config" ) type BasicResponse struct { @@ -201,3 +205,60 @@ func CommandHandler(res http.ResponseWriter, req *http.Request) { exitCode, output := ocis.RunCommand(command, stdIn) sendCmdResponse(res, exitCode, output) } + +func OcisServiceHandler(res http.ResponseWriter, req *http.Request) { + serviceName := req.PathValue("service") + envMap := []string{fmt.Sprintf("OCIS_EXCLUDE_RUN_SERVICES=%s", serviceName)} + + if req.Method == http.MethodPost { + // restart oCIS without service that need to start separately + success, _ := ocis.Restart(envMap) + if success { + var envBody map[string]interface{} + var serviceEnvMap []string + + if req.Body != nil && req.ContentLength > 0 { + var err error + envBody, err = parseJsonBody(req.Body) + if err != nil { + sendResponse(res, http.StatusBadRequest, "Invalid json body") + return + } + } + + for key, value := range envBody { + serviceEnvMap = append(serviceEnvMap, fmt.Sprintf("%s=%v", key, value)) + if strings.HasSuffix(key, "DEBUG_ADDR") { + address := strings.Split(value.(string), ":") + port, _ := strconv.Atoi(address[1]) + config.SetServiceDebugPort(serviceName, port) + } + } + + log.Println(fmt.Sprintf("Starting oCIS service %s...", serviceName)) + + common.Wg.Add(1) + go ocis.StartService(serviceName, serviceEnvMap) + + success := ocis.WaitForServiceStatus(serviceName, true) + if success { + log.Println(fmt.Sprintf("Found Port for %s...", serviceName)) + sendResponse(res, http.StatusOK, fmt.Sprintf("oCIS service %s started successfully", serviceName)) + } else { + sendResponse(res, http.StatusInternalServerError, fmt.Sprintf("Failed to start oCIS service %s", serviceName)) + } + return + } + sendResponse(res, http.StatusInternalServerError, fmt.Sprintf("Failed to restart oCIS without service %s", serviceName)) + return + } else if req.Method == http.MethodDelete { + success, message := ocis.StopService(serviceName) + if success { + sendResponse(res, http.StatusOK, fmt.Sprintf("oCIS service %s stopped successfully", serviceName)) + } else { + sendResponse(res, http.StatusInternalServerError, message) + } + return + } + sendResponse(res, http.StatusMethodNotAllowed, "Method Not Allowed") +} diff --git a/tests/ociswrapper/wrapper/wrapper.go b/tests/ociswrapper/wrapper/wrapper.go index 35b7873134a..8c14006adf0 100644 --- a/tests/ociswrapper/wrapper/wrapper.go +++ b/tests/ociswrapper/wrapper/wrapper.go @@ -27,6 +27,7 @@ func Start(port string) { mux.HandleFunc("/command", handlers.CommandHandler) mux.HandleFunc("/stop", handlers.StopOcisHandler) mux.HandleFunc("/start", handlers.StartOcisHandler) + mux.HandleFunc("/services/{service}", handlers.OcisServiceHandler) httpServer.Handler = mux