Merge branch 'master' into feature/trusted-certs

Author: changlei-li

.github/workflows/main.yml

@@ -32,7 +32,7 @@ jobs:
           ocaml-version: "4.14.2"
           experimental: true
         - runs-on: "ubuntu-22.04"
-          ocaml-version: "5.4.0"
+          ocaml-version: "5.4.1"
           experimental: true
 
     continue-on-error: ${{ matrix.experimental }}

doc/content/design/external-auth-ldaps.md

@@ -180,10 +180,21 @@ client ldap sasl wrapping = <ldaps | seal>
 tls verify peer = ca_and_name_if_available
 tls trust system cas = yes
 tls cafile = /etc/trusted-certs/ca-bundle-[ldaps|general].pem
+tls priority = NONE:+VERS-TLS1.2:+AES-256-GCM:+AES-128-GCM:+AEAD:+ECDHE-RSA:+SIGN-ALL:+GROUP-SECP384R1:+COMP-NULL:%SERVER_PRECEDENCE
 ```
 
 - Switch between `ldap` and `ldaps` will flip `client ldap sasl wrapping` between `seal` and `ldaps`
 - `tls cafile` points to a CA bundle used to verify DC certs. Details refer to 4.1.2
+- `tls priority` is following stunnel TLS configuration, this result to TLS 1.2 with cipher suite TLS_ECDHE_RSA_AES_128_GCM_SHA256, TLS_ECDHE_RSA_AES_256_GCM_SHA384, Windows Server 2008R2 and later as DC support it
+  - NONE: starts with empty set
+  - +VERS-TLS1.2: Enbale TLS 1.2
+  - +AES-256-GCM:+AES-128-GCM: Enable AES-256-GCM and AES-128-GCM cipher
+  - +AEAD: Enable AEAD MAC, required for GCM
+  - +ECDHE-RSA: Enable ECDHE-RSA key exchange
+  - +SIGN-ALL: Enable all signature algorithms, this will limit to TLS1.2
+  - +GROUP-SECP384R1: Enable secp384r1 curve
+  - +COMP-NULL: No compression (required in modern TLS)
+  - %SERVER_PRECEDENCE: Server chooses cipher order
 
 #### 4.1.2 Certificate Selection
 

doc/content/design/migration-compression.md

@@ -0,0 +1,113 @@
+---
+title: Migration Stream Compression inside XenGuest
+layout: default
+design_doc: true
+revision: 1
+status: proposed
+---
+# Migration Stream Compression inside XenGuest
+
+To reduce VM migration time, we want to compress the VM migration data
+stream.  This document describes a new approach that might replace an
+existing feature, described first below.
+
+## Existing Stream Compression
+
+Xapi implements external migration stream compression. This uses an
+external zstd process each to compress (source) and decompress
+(destination) the VM and GPU streams, restively. The compression is
+transparent to the underlying daemons like XenGuest and vgpu/demu. It is
+controlled by an optional boolean parameter `compress` to the VM.migrate
+API/XE call.  When not explicitly provided, a boolean pool parameter
+`Pool.migration_compression` is used. The current default (false) is to
+not use use compression for migration. A pool-level default simplifies
+managing the policy to use for migration compared to a host-level
+default.
+
+This feature requires to communicate the use of compression to the
+receiving host such that it can set up the decompression; this is done
+using cookies. Both VM and GPU memory is compressed during VM migration.
+
+Benchmarking found that this compression scheme is only beneficial when
+using slow networks (like 10Gb Ethernet). It comes with considerable
+internal overhead from using processes and pipe communication for the
+compression and decompression work. That is the reason stream
+compression is not enabled by default.
+
+The main implementation of the feature (with some changes added later)
+is in commit:
+
+* f14fb91137197f24a4784612dd0f2d863ee22fb1
+* CP-39640/CP-39157 Add stream compression for VM migration
+
+## XenGuest-based Stream Compression
+
+This design is about adding an alternative stream compression.  This one
+is implemented inside XenGuest such that the stream it produces is
+internally using compression. XenGuest is not using an external process
+for this and hence incurs less overhead because no process
+intercommunication is required. Benchmarking confirmed the performance
+advantage of this architecture.
+
+The use of compression on the source side needs to be signaled by
+Xenopsd via a command line argument to emu0manager and XenGuest. On the
+receiving side XenGuest will recognise the compressed stream and does
+not require special signaling.
+
+Xapi only permits migration from older to newer versions of the Xapi
+Toolstack. This implies that no older XenGuest version will ever receive
+a potentially compressed stream that it would not be equipped to handle.
+For backward compatibility, xapi still needs to be able to receive a
+compressed migration stream using external compression - even if future
+Xapi implementation chose no longer to create them.
+
+Currently only the stream created by XenGuest would use compression, but
+not the stream containing GPU data created by vgpu/demu. This might
+change in the future when we decide to implement in-stream compression
+there, too.
+
+Theoretically, internal compression implemented in XenGuest and existing
+external compression are independent and could be used both. However,
+this would apply a compression to an already compressed stream, which is
+unlikely to be effective and we don't plan to support this feature.
+
+## API Design
+
+* VM.pool_migrate API and `xe vm-migrate` remain unchanged. Both accept
+  an optional boolean value that indicates whether to use compression or
+  not. In the absence of this parameter, the default is taken from
+  `Pool.migration_compression` (which is currently `false`).
+
+* The compression method used when `Pool.migration_compression=true` is
+  *not* controlled by the API or the customer but is controlled by a
+  default in xenopsd which can be changed in xenopd.conf for testing.
+
+In summary, this means the API and XE CLI interface remain unchanged.
+
+## Implementation
+
+Outside the changes described here, the following other components are
+affected:
+
+* EMU Manager needs to accept a new flag to indicated that compression
+  is used during migration.
+
+* XenGuest needs to accept a flag passed by EMU Manager to use
+  compression.
+
+## Upgrade
+
+Xapi will always accept stream/external compressed migration streams but
+might stop using them for migration.
+
+## Outlook
+
+* We might add internal compression to vGPU migration in the future. The
+  API would be unaffected by this.
+
+* The current design forces all destinations to accept all compression
+  streams because compatibility is not negotiate between source and
+  destination. This is a general problem and not limited to compression.
+  We could expose a general table to clients that lists features
+  supported by xenopsd such that any other xenopsd would send data in
+  the preferred format.

ocaml/idl/datamodel.ml

@@ -8925,6 +8925,23 @@ module Message = struct
       ~params:[(Set (Ref _message), "messages", "Messages to destroy")]
       ~allowed_roles:_R_POOL_OP ()
 
+  let destroy_all =
+    call ~name:"destroy_all" ~lifecycle:[]
+      ~params:
+        [
+          ( Map (String, String)
+          , "filters"
+          , "Optional filters identifying messages to destroy: before (RFC3339 \
+             DateTime, destroy messages dated before this timestamp), after \
+             (RFC3339 DateTime, destroy messages dated after this timestamp), \
+             and priority (int, only destroy messages with this priority). All \
+             provided conditions must be met (logical AND). If no filters are \
+             provided, all messages are destroyed. If no timezone is specified \
+             in a timestamp, UTC is assumed."
+          )
+        ]
+      ~allowed_roles:_R_POOL_OP ()
+
   let get_all =
     call ~name:"get_all"
       ~lifecycle:[(Published, rel_orlando, "")]
@@ -9012,6 +9029,7 @@ module Message = struct
           create
         ; destroy
         ; destroy_many
+        ; destroy_all
         ; get
         ; get_all
         ; get_since

ocaml/idl/datamodel_errors.ml

@@ -893,7 +893,11 @@ let _ =
        again."
     () ;
   error Api_errors.pool_joining_pool_cannot_enable_clustering_on_vlan_network
-    ["vlan"] ~doc:"The remote pool cannot enable clustering on vlan network" () ;
+    ["vlan"]
+    ~doc:
+      "The host attempted to join a pool with clustering enabled on a \
+       non-management VLAN network, which is not supported."
+    () ;
   error Api_errors.pool_joining_host_must_have_only_one_IP_on_clustering_network
     []
     ~doc:

ocaml/idl/datamodel_lifecycle.ml

@@ -233,6 +233,8 @@ let prototyped_of_message = function
       Some "24.14.0"
   | "PCI", "disable_dom0_access" ->
       Some "24.14.0"
+  | "message", "destroy_all" ->
+      Some "26.5.0-next"
   | "message", "destroy_many" ->
       Some "22.19.0"
   | "VTPM", "set_contents" ->

ocaml/libs/stunnel/stunnel.ml

@@ -152,6 +152,8 @@ let pool =
   ; crl_dir= None
   }
 
+let world = {appliance with cert_bundle_path= "/etc/ssl/certs/ca-bundle.crt"}
+
 let external_host ext_host_cert_file =
   {
     sni= None

ocaml/libs/stunnel/stunnel.mli

@@ -60,6 +60,8 @@ val appliance : verification_config
 
 val pool : verification_config
 
+val world : verification_config
+
 val external_host : string -> verification_config
 
 val with_connect :

ocaml/libs/stunnel/stunnel_client.ml

@@ -33,6 +33,8 @@ let pool () = get_verification_config Stunnel.pool
 
 let appliance () = get_verification_config Stunnel.appliance
 
+let world () = get_verification_config Stunnel.world
+
 let external_host cert_file =
   Stunnel.external_host cert_file |> get_verification_config
 

ocaml/libs/stunnel/stunnel_client.mli

@@ -24,6 +24,11 @@ val appliance : unit -> Stunnel.verification_config option
 (** [appliance ()] returns the configuration that's meant to be used to connect
     to appliances providing services, like WLB or a licensing server. *)
 
+val world : unit -> Stunnel.verification_config option
+(** [world ()] returns the configuration that performs chain
+    verification using the system's default CA trust store
+    (/etc/ssl/certs/ca-bundle.crt). *)
+
 val external_host : string -> Stunnel.verification_config option
 (** [external_host path] returns the configuration that's meant to be used to connect to
     a xapi hosts outside the pool. This is useful, for example, to provide an