Skip to content

Commit da53638

Browse files
oleksandr-ncoleksandr-nc
committed
feat: add Kubernetes deployment support
Add KubernetesActions deploy backend that manages ExApp lifecycle (deploy, expose, remove) via HaRP's Kubernetes API endpoints. Wire K8s flow into CLI commands (register/unregister daemon and ExApp) and pass kubernetes deploy config from the Vue frontend. Signed-off-by: Oleksander Piskun <oleksandr2088@icloud.com>
1 parent 821785a commit da53638

13 files changed

Lines changed: 1171 additions & 46 deletions

‎js/app_api-adminSettings.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎js/app_api-adminSettings.js.map‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/Command/Daemon/RegisterDaemon.php‎

Lines changed: 91 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,27 @@ protected function configure(): void {
5050
$this->addOption('harp_docker_socket_port', null, InputOption::VALUE_REQUIRED, '\'remotePort\' of the FRP client of the remote Docker socket proxy. There is one included in the harp container so this can be skipped for default setups.', '24000');
5151
$this->addOption('harp_exapp_direct', null, InputOption::VALUE_NONE, 'Flag for the advanced setups only. Disables the FRP tunnel between ExApps and HaRP.');
5252

53+
// Kubernetes options
54+
$this->addOption('k8s', null, InputOption::VALUE_NONE, 'Flag to indicate Kubernetes daemon (uses kubernetes-install deploy ID). Requires --harp flag.');
55+
$this->addOption('k8s_expose_type', null, InputOption::VALUE_REQUIRED, 'Kubernetes Service type: nodeport|clusterip|loadbalancer|manual (default: clusterip)', 'clusterip');
56+
$this->addOption('k8s_node_port', null, InputOption::VALUE_REQUIRED, 'Optional NodePort (30000-32767) for nodeport expose type');
57+
$this->addOption('k8s_upstream_host', null, InputOption::VALUE_REQUIRED, 'Override upstream host for HaRP to reach ExApps. Required for manual expose type.');
58+
$this->addOption('k8s_external_traffic_policy', null, InputOption::VALUE_REQUIRED, 'Cluster|Local for NodePort/LoadBalancer Service types');
59+
$this->addOption('k8s_load_balancer_ip', null, InputOption::VALUE_REQUIRED, 'Optional LoadBalancer IP for loadbalancer expose type');
60+
$this->addOption('k8s_node_address_type', null, InputOption::VALUE_REQUIRED, 'InternalIP|ExternalIP for auto node selection (default: InternalIP)', 'InternalIP');
61+
5362
$this->addUsage('harp_proxy_docker "Harp Proxy (Docker)" "docker-install" "http" "appapi-harp:8780" "http://nextcloud.local" --net nextcloud --harp --harp_frp_address "appapi-harp:8782" --harp_shared_key "some_very_secure_password" --set-default --compute_device=cuda');
5463
$this->addUsage('harp_proxy_host "Harp Proxy (Host)" "docker-install" "http" "localhost:8780" "http://nextcloud.local" --harp --harp_frp_address "localhost:8782" --harp_shared_key "some_very_secure_password" --set-default --compute_device=cuda');
5564
$this->addUsage('manual_install_harp "Harp Manual Install" "manual-install" "http" "appapi-harp:8780" "http://nextcloud.local" --net nextcloud --harp --harp_frp_address "appapi-harp:8782" --harp_shared_key "some_very_secure_password"');
5665
$this->addUsage('docker_install "Docker Socket Proxy" "docker-install" "http" "nextcloud-appapi-dsp:2375" "http://nextcloud.local" --net=nextcloud --set-default --compute_device=cuda');
5766
$this->addUsage('manual_install "Manual Install" "manual-install" "http" null "http://nextcloud.local"');
5867
$this->addUsage('local_docker "Docker Local" "docker-install" "http" "/var/run/docker.sock" "http://nextcloud.local" --net=nextcloud');
5968
$this->addUsage('local_docker "Docker Local" "docker-install" "http" "/var/run/docker.sock" "http://nextcloud.local" --net=nextcloud --set-default --compute_device=cuda');
69+
70+
// Kubernetes usage examples
71+
$this->addUsage('k8s_daemon "Kubernetes HaRP" "kubernetes-install" "http" "harp.nextcloud.svc:8780" "http://nextcloud.local" --harp --harp_shared_key "secret" --harp_frp_address "harp.nextcloud.svc:8782" --k8s');
72+
$this->addUsage('k8s_daemon_nodeport "K8s NodePort" "kubernetes-install" "http" "harp.example.com:8780" "http://nextcloud.local" --harp --harp_shared_key "secret" --harp_frp_address "harp.example.com:8782" --k8s --k8s_expose_type=nodeport --k8s_upstream_host="k8s-node.example.com"');
73+
$this->addUsage('k8s_daemon_lb "K8s LoadBalancer" "kubernetes-install" "http" "harp.example.com:8780" "http://nextcloud.local" --harp --harp_shared_key "secret" --harp_frp_address "harp.example.com:8782" --k8s --k8s_expose_type=loadbalancer');
6074
}
6175

6276
protected function execute(InputInterface $input, OutputInterface $output): int {
@@ -67,6 +81,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
6781
$host = $input->getArgument('host');
6882
$nextcloudUrl = $input->getArgument('nextcloud_url');
6983
$isHarp = $input->getOption('harp');
84+
$isK8s = $input->getOption('k8s');
7085

7186
if (($protocol !== 'http') && ($protocol !== 'https')) {
7287
$output->writeln('Value error: The protocol must be `http` or `https`.');
@@ -81,6 +96,67 @@ protected function execute(InputInterface $input, OutputInterface $output): int
8196
return 1;
8297
}
8398

99+
// Kubernetes validation
100+
if ($isK8s) {
101+
if (!$isHarp) {
102+
$output->writeln('Value error: Kubernetes daemon (--k8s) requires --harp flag. K8s always uses HaRP.');
103+
return 1;
104+
}
105+
// Override accepts-deploy-id for K8s
106+
if ($acceptsDeployId !== 'kubernetes-install') {
107+
$output->writeln('<comment>Note: --k8s flag detected. Overriding accepts-deploy-id to "kubernetes-install".</comment>');
108+
$acceptsDeployId = 'kubernetes-install';
109+
}
110+
111+
$k8sExposeType = $input->getOption('k8s_expose_type');
112+
$validExposeTypes = ['nodeport', 'clusterip', 'loadbalancer', 'manual'];
113+
if (!in_array($k8sExposeType, $validExposeTypes)) {
114+
$output->writeln(sprintf('Value error: Invalid k8s_expose_type "%s". Must be one of: %s', $k8sExposeType, implode(', ', $validExposeTypes)));
115+
return 1;
116+
}
117+
118+
$k8sNodePort = $input->getOption('k8s_node_port');
119+
if ($k8sNodePort !== null) {
120+
$k8sNodePort = (int)$k8sNodePort;
121+
if ($k8sExposeType !== 'nodeport') {
122+
$output->writeln('Value error: --k8s_node_port is only valid with --k8s_expose_type=nodeport');
123+
return 1;
124+
}
125+
if ($k8sNodePort < 30000 || $k8sNodePort > 32767) {
126+
$output->writeln('Value error: --k8s_node_port must be between 30000 and 32767');
127+
return 1;
128+
}
129+
}
130+
131+
$k8sLoadBalancerIp = $input->getOption('k8s_load_balancer_ip');
132+
if ($k8sLoadBalancerIp !== null && $k8sExposeType !== 'loadbalancer') {
133+
$output->writeln('Value error: --k8s_load_balancer_ip is only valid with --k8s_expose_type=loadbalancer');
134+
return 1;
135+
}
136+
137+
$k8sUpstreamHost = $input->getOption('k8s_upstream_host');
138+
if ($k8sExposeType === 'manual' && $k8sUpstreamHost === null) {
139+
$output->writeln('Value error: --k8s_upstream_host is required for --k8s_expose_type=manual');
140+
return 1;
141+
}
142+
143+
$k8sExternalTrafficPolicy = $input->getOption('k8s_external_traffic_policy');
144+
if ($k8sExternalTrafficPolicy !== null) {
145+
$validPolicies = ['Cluster', 'Local'];
146+
if (!in_array($k8sExternalTrafficPolicy, $validPolicies)) {
147+
$output->writeln(sprintf('Value error: Invalid k8s_external_traffic_policy "%s". Must be one of: %s', $k8sExternalTrafficPolicy, implode(', ', $validPolicies)));
148+
return 1;
149+
}
150+
}
151+
152+
$k8sNodeAddressType = $input->getOption('k8s_node_address_type');
153+
$validNodeAddressTypes = ['InternalIP', 'ExternalIP'];
154+
if (!in_array($k8sNodeAddressType, $validNodeAddressTypes)) {
155+
$output->writeln(sprintf('Value error: Invalid k8s_node_address_type "%s". Must be one of: %s', $k8sNodeAddressType, implode(', ', $validNodeAddressTypes)));
156+
return 1;
157+
}
158+
}
159+
84160
if ($acceptsDeployId === 'manual-install' && !$isHarp && str_contains($host, ':')) {
85161
$output->writeln('<comment>Warning: The host contains a port, which will be ignored for manual-install daemons. The ExApp\'s port from --json-info will be used instead.</comment>');
86162
}
@@ -94,18 +170,31 @@ protected function execute(InputInterface $input, OutputInterface $output): int
94170
? $input->getOption('harp_shared_key')
95171
: $input->getOption('haproxy_password') ?? '';
96172

173+
$defaultNet = $isK8s ? 'bridge' : 'host';
97174
$deployConfig = [
98-
'net' => $input->getOption('net') ?? 'host',
175+
'net' => $input->getOption('net') ?? $defaultNet,
99176
'nextcloud_url' => $nextcloudUrl,
100177
'haproxy_password' => $secret,
101178
'computeDevice' => $this->buildComputeDevice($input->getOption('compute_device') ?? 'cpu'),
102179
'harp' => null,
180+
'kubernetes' => null,
103181
];
104182
if ($isHarp) {
105183
$deployConfig['harp'] = [
106184
'frp_address' => $input->getOption('harp_frp_address') ?? '',
107185
'docker_socket_port' => $input->getOption('harp_docker_socket_port'),
108-
'exapp_direct' => (bool)$input->getOption('harp_exapp_direct'),
186+
'exapp_direct' => $isK8s ? true : (bool)$input->getOption('harp_exapp_direct'),
187+
];
188+
}
189+
if ($isK8s) {
190+
$k8sNodePort = $input->getOption('k8s_node_port');
191+
$deployConfig['kubernetes'] = [
192+
'expose_type' => $input->getOption('k8s_expose_type') ?? 'clusterip',
193+
'node_port' => $k8sNodePort !== null ? (int)$k8sNodePort : null,
194+
'upstream_host' => $input->getOption('k8s_upstream_host'),
195+
'external_traffic_policy' => $input->getOption('k8s_external_traffic_policy'),
196+
'load_balancer_ip' => $input->getOption('k8s_load_balancer_ip'),
197+
'node_address_type' => $input->getOption('k8s_node_address_type') ?? 'InternalIP',
109198
];
110199
}
111200

‎lib/Command/ExApp/Register.php‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
use OCA\AppAPI\AppInfo\Application;
1313
use OCA\AppAPI\DeployActions\DockerActions;
14+
use OCA\AppAPI\DeployActions\KubernetesActions;
1415
use OCA\AppAPI\DeployActions\ManualActions;
1516
use OCA\AppAPI\Fetcher\ExAppArchiveFetcher;
1617
use OCA\AppAPI\Service\AppAPIService;
@@ -33,6 +34,7 @@ public function __construct(
3334
private readonly DaemonConfigService $daemonConfigService,
3435
private readonly DockerActions $dockerActions,
3536
private readonly ManualActions $manualActions,
37+
private readonly KubernetesActions $kubernetesActions,
3638
private readonly IAppConfig $appConfig,
3739
private readonly ExAppService $exAppService,
3840
private readonly ISecureRandom $random,
@@ -132,6 +134,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
132134
$actionsDeployIds = [
133135
$this->dockerActions->getAcceptsDeployId(),
134136
$this->manualActions->getAcceptsDeployId(),
137+
$this->kubernetesActions->getAcceptsDeployId(),
135138
];
136139
if (!in_array($daemonConfig->getAcceptsDeployId(), $actionsDeployIds)) {
137140
$this->logger->error(sprintf('Daemon config %s actions for %s not found.', $daemonConfigName, $daemonConfig->getAcceptsDeployId()));
@@ -166,6 +169,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
166169
}
167170

168171
$auth = [];
172+
$harpK8sUrl = null;
173+
$k8sRoles = [];
169174
if ($daemonConfig->getAcceptsDeployId() === $this->dockerActions->getAcceptsDeployId()) {
170175
$deployParams = $this->dockerActions->buildDeployParams($daemonConfig, $appInfo);
171176
if (boolval($exApp->getDeployConfig()['harp'] ?? false)) {
@@ -200,6 +205,53 @@ protected function execute(InputInterface $input, OutputInterface $output): int
200205
(int)explode('=', $deployParams['container_params']['env'][6])[1],
201206
$auth,
202207
);
208+
} elseif ($daemonConfig->getAcceptsDeployId() === $this->kubernetesActions->getAcceptsDeployId()) {
209+
$deployParams = $this->kubernetesActions->buildDeployParams($daemonConfig, $appInfo);
210+
$this->kubernetesActions->initGuzzleClient($daemonConfig);
211+
$harpK8sUrl = $this->kubernetesActions->buildHarpK8sUrl($daemonConfig);
212+
$k8sRoles = $deployParams['k8s_service_roles'] ?? [];
213+
$deployResult = $this->kubernetesActions->deployExApp($exApp, $daemonConfig, $deployParams);
214+
if ($deployResult) {
215+
$this->logger->error(sprintf('ExApp %s K8s deployment failed. Error: %s', $appId, $deployResult));
216+
if ($outputConsole) {
217+
$output->writeln(sprintf('ExApp %s K8s deployment failed. Error: %s', $appId, $deployResult));
218+
}
219+
$this->exAppService->setStatusError($exApp, $deployResult);
220+
$this->kubernetesActions->cleanupResources($harpK8sUrl, $appId, $k8sRoles);
221+
$this->_unregisterExApp($appId, $isTestDeployMode);
222+
return 1;
223+
}
224+
225+
// For K8s, expose the ExApp (create Service) and get upstream endpoint
226+
$k8sConfig = $daemonConfig->getDeployConfig()['kubernetes'] ?? [];
227+
if (!empty($k8sRoles)) {
228+
$exposeResult = $this->kubernetesActions->exposeExAppRoles(
229+
$harpK8sUrl, $appId, (int)$appInfo['port'], $k8sConfig, $k8sRoles
230+
);
231+
} else {
232+
$exposeResult = $this->kubernetesActions->exposeExApp(
233+
$harpK8sUrl, $appId, (int)$appInfo['port'], $k8sConfig
234+
);
235+
}
236+
if (isset($exposeResult['error'])) {
237+
$this->logger->error(sprintf('ExApp %s K8s expose failed. Error: %s', $appId, $exposeResult['error']));
238+
if ($outputConsole) {
239+
$output->writeln(sprintf('ExApp %s K8s expose failed. Error: %s', $appId, $exposeResult['error']));
240+
}
241+
$this->exAppService->setStatusError($exApp, $exposeResult['error']);
242+
$this->kubernetesActions->cleanupResources($harpK8sUrl, $appId, $k8sRoles);
243+
$this->_unregisterExApp($appId, $isTestDeployMode);
244+
return 1;
245+
}
246+
247+
$exAppUrl = $this->kubernetesActions->resolveExAppUrl(
248+
$appId,
249+
$daemonConfig->getProtocol(),
250+
$daemonConfig->getHost(),
251+
$daemonConfig->getDeployConfig(),
252+
(int)$appInfo['port'],
253+
$auth,
254+
);
203255
} else {
204256
$this->manualActions->deployExApp($exApp, $daemonConfig);
205257
$exAppUrl = $this->manualActions->resolveExAppUrl(
@@ -218,6 +270,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
218270
$output->writeln(sprintf('ExApp %s heartbeat check failed. Make sure that Nextcloud instance and ExApp can reach it other.', $appId));
219271
}
220272
$this->exAppService->setStatusError($exApp, 'Heartbeat check failed');
273+
if ($harpK8sUrl !== null) {
274+
$this->kubernetesActions->cleanupResources($harpK8sUrl, $appId, $k8sRoles);
275+
$this->_unregisterExApp($appId, $isTestDeployMode);
276+
}
221277
return 1;
222278
}
223279
$this->logger->info(sprintf('ExApp %s deployed successfully.', $appId));

‎lib/Command/ExApp/Unregister.php‎

Lines changed: 68 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@
1010
namespace OCA\AppAPI\Command\ExApp;
1111

1212
use OCA\AppAPI\DeployActions\DockerActions;
13+
use OCA\AppAPI\DeployActions\KubernetesActions;
1314

1415
use OCA\AppAPI\Service\AppAPIService;
1516
use OCA\AppAPI\Service\DaemonConfigService;
17+
use OCA\AppAPI\Service\ExAppDeployOptionsService;
1618
use OCA\AppAPI\Service\ExAppService;
1719
use Symfony\Component\Console\Command\Command;
1820
use Symfony\Component\Console\Input\InputArgument;
@@ -26,7 +28,9 @@ public function __construct(
2628
private readonly AppAPIService $service,
2729
private readonly DaemonConfigService $daemonConfigService,
2830
private readonly DockerActions $dockerActions,
31+
private readonly KubernetesActions $kubernetesActions,
2932
private readonly ExAppService $exAppService,
33+
private readonly ExAppDeployOptionsService $exAppDeployOptionsService,
3034
) {
3135
parent::__construct();
3236
}
@@ -95,51 +99,81 @@ protected function execute(InputInterface $input, OutputInterface $output): int
9599
return 1;
96100
}
97101
}
98-
if ($daemonConfig->getAcceptsDeployId() === $this->dockerActions->getAcceptsDeployId()) {
99-
$this->dockerActions->initGuzzleClient($daemonConfig);
100-
101-
if (boolval($exApp->getDeployConfig()['harp'] ?? false)) {
102-
if ($this->dockerActions->removeExApp($this->dockerActions->buildDockerUrl($daemonConfig), $exApp->getAppid(), removeData: $rmData)) {
103-
if (!$silent) {
104-
$output->writeln(sprintf('Failed to remove ExApp %s', $appId));
105-
$output->writeln('Hint: If the container was already removed manually, you can use the --force option to fully remove it from AppAPI.');
106-
}
107-
if (!$force) {
108-
return 1;
102+
if ($daemonConfig !== null) {
103+
if ($daemonConfig->getAcceptsDeployId() === $this->dockerActions->getAcceptsDeployId()) {
104+
$this->dockerActions->initGuzzleClient($daemonConfig);
105+
106+
if (boolval($exApp->getDeployConfig()['harp'] ?? false)) {
107+
if ($this->dockerActions->removeExApp($this->dockerActions->buildDockerUrl($daemonConfig), $exApp->getAppid(), removeData: $rmData)) {
108+
if (!$silent) {
109+
$output->writeln(sprintf('Failed to remove ExApp %s', $appId));
110+
$output->writeln('Hint: If the container was already removed manually, you can use the --force option to fully remove it from AppAPI.');
111+
}
112+
if (!$force) {
113+
return 1;
114+
}
115+
} else {
116+
if (!$silent) {
117+
$output->writeln(sprintf('ExApp %s successfully removed', $appId));
118+
}
109119
}
110120
} else {
111-
if (!$silent) {
112-
$output->writeln(sprintf('ExApp %s successfully removed', $appId));
121+
$containerName = $this->dockerActions->buildExAppContainerName($appId);
122+
$removeResult = $this->dockerActions->removeContainer(
123+
$this->dockerActions->buildDockerUrl($daemonConfig), $containerName
124+
);
125+
if ($removeResult) {
126+
if (!$silent) {
127+
$output->writeln(sprintf('Failed to remove ExApp %s container', $appId));
128+
$output->writeln(sprintf('Hint: If the container "%s" was already removed manually, you can use the --force option to fully remove it from AppAPI.', $containerName));
129+
}
130+
if (!$force) {
131+
return 1;
132+
}
133+
} elseif (!$silent) {
134+
$output->writeln(sprintf('ExApp %s container successfully removed', $appId));
135+
}
136+
if ($rmData) {
137+
$volumeName = $this->dockerActions->buildExAppVolumeName($appId);
138+
$removeVolumeResult = $this->dockerActions->removeVolume(
139+
$this->dockerActions->buildDockerUrl($daemonConfig), $volumeName
140+
);
141+
if (!$silent) {
142+
if (isset($removeVolumeResult['error'])) {
143+
$output->writeln(sprintf('Failed to remove ExApp %s volume: %s', $appId, $volumeName));
144+
} else {
145+
$output->writeln(sprintf('ExApp %s data volume successfully removed', $appId));
146+
}
147+
}
113148
}
114149
}
115-
} else {
116-
$containerName = $this->dockerActions->buildExAppContainerName($appId);
117-
$removeResult = $this->dockerActions->removeContainer(
118-
$this->dockerActions->buildDockerUrl($daemonConfig), $containerName
119-
);
150+
} elseif ($daemonConfig->getAcceptsDeployId() === $this->kubernetesActions->getAcceptsDeployId()) {
151+
$this->kubernetesActions->initGuzzleClient($daemonConfig);
152+
$harpK8sUrl = $this->kubernetesActions->buildHarpK8sUrl($daemonConfig);
153+
154+
// Check for stored multi-role configuration
155+
$rolesOption = $this->exAppDeployOptionsService->getDeployOption($exApp->getAppid(), 'k8s_service_roles');
156+
$roles = $rolesOption !== null ? $rolesOption->getValue() : [];
157+
158+
if (!empty($roles) && is_array($roles)) {
159+
$removeResult = $this->kubernetesActions->removeAllRoles(
160+
$harpK8sUrl, $exApp->getAppid(), $roles, removeData: $rmData
161+
);
162+
} else {
163+
$removeResult = $this->kubernetesActions->removeExApp(
164+
$harpK8sUrl, $exApp->getAppid(), removeData: $rmData
165+
);
166+
}
120167
if ($removeResult) {
121168
if (!$silent) {
122-
$output->writeln(sprintf('Failed to remove ExApp %s container', $appId));
123-
$output->writeln(sprintf('Hint: If the container "%s" was already removed manually, you can use the --force option to fully remove it from AppAPI.', $containerName));
169+
$output->writeln(sprintf('Failed to remove K8s ExApp %s: %s', $appId, $removeResult));
170+
$output->writeln('Hint: If the K8s deployment was already removed manually, use --force to remove from AppAPI.');
124171
}
125172
if (!$force) {
126173
return 1;
127174
}
128175
} elseif (!$silent) {
129-
$output->writeln(sprintf('ExApp %s container successfully removed', $appId));
130-
}
131-
if ($rmData) {
132-
$volumeName = $this->dockerActions->buildExAppVolumeName($appId);
133-
$removeVolumeResult = $this->dockerActions->removeVolume(
134-
$this->dockerActions->buildDockerUrl($daemonConfig), $volumeName
135-
);
136-
if (!$silent) {
137-
if (isset($removeVolumeResult['error'])) {
138-
$output->writeln(sprintf('Failed to remove ExApp %s volume: %s', $appId, $volumeName));
139-
} else {
140-
$output->writeln(sprintf('ExApp %s data volume successfully removed', $appId));
141-
}
142-
}
176+
$output->writeln(sprintf('ExApp %s K8s resources successfully removed', $appId));
143177
}
144178
}
145179
}

0 commit comments

Comments
 (0)