Emit string and byte-string constants (#927, #546)

`&str` and `&CStr` constants were silently dropped; they are now emitted
as C string literals. A scalar `&str`/`&CStr` becomes `const char NAME[]`
(an unsized array) in the C++/Cython typed forms, so `sizeof` gives the
length at compile time and the declaration mirrors its initializer; in C
it stays a bare `#define`. An array of them (`[&CStr; N]`) instead maps to
`const char *[N]`, since ragged element lengths can't form a `char[N][]`.

Byte-string constants (`b"..."`) are emitted as `uint8_t[]` array
initializers rather than string literals, so non-printable and
non-NUL-terminated data survives. The declared type (`&[u8; N]` or the
unsized `&[u8]`) doesn't load into a bare array, so `load_byte_string`
derives both the type and the initializer from the literal bytes, which
also makes the sized and unsized forms behave identically. An empty byte
string yields the same zero-length array as any other empty array
constant; it is not special-cased.

`&str`/`&CStr` are fat pointers, so their C form is not ABI-compatible
with the Rust type; what is emitted is the string *value*, which is what a
C consumer of these constants wants. The rewrite is confined to constant
definitions and never applied to fields or arguments.

The C string escaper renders non-ASCII UTF-8 bytes as `\xNN` (not Rust's
`\u{...}`, which is invalid C), escapes `?` to prevent trigraph
formation, and splits a literal when a `\xNN` escape is followed by a
hex-digit character so the escape stays one byte.

The three const-dedup checks that suppress a doubled `const` for pointer
constants are consolidated into `write_field_prepends_const`, which also
recurses through arrays so arrays of const pointers dedup correctly.

Author: cf-rhett

CHANGES

@@ -1,5 +1,8 @@
 # unreleased
 
+      * Emit `&str`, `&CStr`, and arrays thereof as C string literals, and byte-string
+        (`b"..."`) constants as `uint8_t[]` arrays, instead of dropping them.
+
 # 0.29.4
 
       * Support constant enums and arrays.

src/bindgen/ir/constant.rs

@@ -4,6 +4,7 @@
 
 use std::borrow::Cow;
 use std::collections::HashMap;
+use std::fmt::Write as _;
 use std::io::Write;
 
 use syn::ext::IdentExt;
@@ -13,8 +14,8 @@ use crate::bindgen::config::{Config, Language};
 use crate::bindgen::declarationtyperesolver::DeclarationTypeResolver;
 use crate::bindgen::dependencies::Dependencies;
 use crate::bindgen::ir::{
-    AnnotationSet, Cfg, ConditionWrite, Documentation, GenericParams, Item, ItemContainer, Path,
-    Struct, ToCondition, Type,
+    AnnotationSet, Cfg, ConditionWrite, ConstExpr, Documentation, GenericParams, IntKind, Item,
+    ItemContainer, Path, PrimitiveType, Struct, ToCondition, Type,
 };
 use crate::bindgen::language_backend::LanguageBackend;
 use crate::bindgen::library::Library;
@@ -29,10 +30,140 @@ fn member_to_ident(member: &syn::Member) -> String {
     }
 }
 
+/// Format a byte slice as a double-quoted string literal valid in both C and C++.
+///
+/// Printable ASCII passes through; `"`, `\`, `?`, and the common whitespace
+/// controls use their named escapes; every other byte (other control characters
+/// and the raw bytes of non-ASCII UTF-8) becomes `\xNN`. Rust's own `{:?}` /
+/// `escape_default` is unsuitable here because it renders non-ASCII as `\u{...}`,
+/// which is not valid C.
+///
+/// A `\xNN` escape in C greedily consumes every following hex digit, so when such an
+/// escape is followed by a literal hex-digit character the literal is split
+/// (`"...\xNN" "f..."`); adjacent string literals concatenate, keeping the escape a
+/// single byte.
+fn to_c_string_literal(bytes: &[u8]) -> String {
+    let mut out = String::with_capacity(bytes.len() * 4 + 2);
+    out.push('"');
+    let mut prev_was_hex_escape = false;
+    for &byte in bytes {
+        match byte {
+            b'"' => out.push_str("\\\""),
+            b'\\' => out.push_str("\\\\"),
+            b'\n' => out.push_str("\\n"),
+            b'\r' => out.push_str("\\r"),
+            b'\t' => out.push_str("\\t"),
+            // Escape `?` so no `??x` trigraph can form (trigraphs are translated
+            // even inside string literals in C before C23 / C++ before C++17).
+            b'?' => out.push_str("\\?"),
+            0x20..=0x7e => {
+                if prev_was_hex_escape && byte.is_ascii_hexdigit() {
+                    out.push_str("\" \"");
+                }
+                out.push(byte as char);
+            }
+            _ => {
+                let _ = write!(out, "\\x{byte:02x}");
+                prev_was_hex_escape = true;
+                continue;
+            }
+        }
+        prev_was_hex_escape = false;
+    }
+    out.push('"');
+    out
+}
+
+/// Rewrite the Rust standard string reference types (`&str`, `&CStr`) a string
+/// constant carries into the C `char` pointer they are exposed as.
+///
+/// Whether `ty` is the `str` / `CStr` path a `&str` / `&CStr` points to. Neither
+/// is a real C type, so a constant of that pointee is lowered to `char`.
+fn is_string_path(ty: &Type) -> bool {
+    matches!(ty, Type::Path(path) if path.generics().is_empty() && matches!(path.name(), "str" | "CStr"))
+}
+
+/// Rewrites a `&str` / `&CStr` constant's type into its C form.
+///
+/// A scalar `&str` / `&CStr` becomes `char[]` -- an array, not a pointer -- so
+/// `sizeof` yields the length at compile time and the declaration mirrors its
+/// string-literal initializer. Inside an array each element instead stays a
+/// `char *`: a `[&CStr; N]` maps to `const char *[N]`, since ragged element
+/// lengths can't form a `char[N][]`.
+///
+/// In C the constant emits as a bare `#define`, so the type is unused; this only
+/// shapes the C++ typed-constant and Cython declarations.
+fn rewrite_string_reference_type(ty: &mut Type) {
+    if matches!(ty, Type::Ptr { ty, .. } if is_string_path(ty)) {
+        *ty = Type::Array(
+            Box::new(Type::Primitive(PrimitiveType::Char)),
+            ConstExpr::Value(String::new()),
+        );
+    } else if let Type::Array(inner, _) = ty {
+        reduce_string_reference_pointee(inner);
+    }
+}
+
+/// Reduces the `str` / `CStr` pointee of a `&str` / `&CStr` array element to
+/// `char`, keeping it a pointer, and recurses through nested arrays.
+fn reduce_string_reference_pointee(ty: &mut Type) {
+    match ty {
+        Type::Ptr { ty, .. } if is_string_path(ty) => {
+            **ty = Type::Primitive(PrimitiveType::Char);
+        }
+        Type::Ptr { ty, .. } | Type::Array(ty, _) => reduce_string_reference_pointee(ty),
+        _ => {}
+    }
+}
+
+/// Whether `write_field` will itself render a leading `const` for `ty`, so the
+/// constant writers must not prepend a second one. True for a const pointer and
+/// for an array whose element is (recursively) a const pointer.
+fn write_field_prepends_const(ty: &Type) -> bool {
+    match ty {
+        Type::Ptr { is_const, .. } => *is_const,
+        Type::Array(inner, _) => write_field_prepends_const(inner),
+        _ => false,
+    }
+}
+
+/// Loads a byte-string literal (`b"..."`) as a `uint8_t[N]` constant.
+///
+/// Returns the constant's type and initializer, or `None` if `expr` is not a
+/// byte string. A byte string is exposed as a `uint8_t` array rather than a C
+/// string literal so that non-printable and non-NUL-terminated data survives (a
+/// `b"..."` is not NUL-terminated and may contain interior NULs). Its declared
+/// type (`&[u8; N]` or the unsized `&[u8]`) does not load into a bare array, so
+/// both the type and the initializer are derived together from the literal
+/// bytes -- which also makes the sized and unsized forms behave identically.
+fn load_byte_string(expr: &syn::Expr) -> Option<(Type, Literal)> {
+    let bytes = match expr {
+        syn::Expr::Lit(syn::ExprLit {
+            lit: syn::Lit::ByteStr(value),
+            ..
+        }) => value.value(),
+        _ => return None,
+    };
+
+    let ty = Type::Array(
+        Box::new(Type::Primitive(PrimitiveType::Integer {
+            zeroable: true,
+            signed: false,
+            kind: IntKind::B8,
+        })),
+        ConstExpr::Value(bytes.len().to_string()),
+    );
+    let lit = Literal::Array {
+        items: bytes
+            .iter()
+            .map(|byte| Literal::Expr(byte.to_string()))
+            .collect(),
+    };
+    Some((ty, lit))
+}
+
 // TODO: Maybe add support to more std associated constants.
 pub(crate) fn to_known_assoc_constant(associated_to: &Path, name: &str) -> Option<String> {
-    use crate::bindgen::ir::{IntKind, PrimitiveType};
-
     if name != "MAX" && name != "MIN" {
         return None;
     }
@@ -413,7 +544,15 @@ impl Literal {
                         Ok(Literal::Expr(value.base10_digits().to_string()))
                     }
                     syn::Lit::Bool(ref value) => Ok(Literal::Expr(format!("{}", value.value))),
-                    // TODO: Add support for byte string and Verbatim
+                    syn::Lit::Str(ref value) => {
+                        Ok(Literal::Expr(to_c_string_literal(value.value().as_bytes())))
+                    }
+                    syn::Lit::CStr(ref value) => {
+                        Ok(Literal::Expr(to_c_string_literal(value.value().to_bytes())))
+                    }
+                    // Byte-string literals are handled by `load_byte_string`, which
+                    // synthesizes the array type; nested byte strings stay unsupported.
+                    // TODO: Add support for Verbatim
                     _ => Err(format!("Unsupported literal expression. {:?}", *lit)),
                 }
             }
@@ -562,16 +701,16 @@ impl Constant {
         attrs: &[syn::Attribute],
         associated_to: Option<Path>,
     ) -> Result<Constant, String> {
-        let ty = Type::load(ty)?;
-        let mut ty = match ty {
-            Some(ty) => ty,
+        let (mut ty, mut lit) = match load_byte_string(expr) {
+            Some(byte_string) => byte_string,
             None => {
-                return Err("Cannot have a zero sized const definition.".to_owned());
+                let mut ty = Type::load(ty)?
+                    .ok_or_else(|| "Cannot have a zero sized const definition.".to_owned())?;
+                rewrite_string_reference_type(&mut ty);
+                (ty, Literal::load(expr)?)
             }
         };
 
-        let mut lit = Literal::load(expr)?;
-
         if let Some(ref associated_to) = associated_to {
             ty.replace_self_with(associated_to);
             lit.replace_self_with(associated_to);
@@ -690,7 +829,7 @@ impl Constant {
 
         let condition = self.cfg.to_condition(config);
         condition.write_before(config, out);
-        if let Type::Ptr { is_const: true, .. } = self.ty {
+        if write_field_prepends_const(&self.ty) {
             out.write("static ");
         } else {
             out.write("static const ");
@@ -790,9 +929,7 @@ impl Constant {
                     out.write(if in_body { "inline " } else { "static " });
                 }
 
-                if let Type::Ptr { is_const: true, .. } = self.ty {
-                    // Nothing.
-                } else {
+                if !write_field_prepends_const(&self.ty) {
                     out.write("const ");
                 }
                 crate::bindgen::cdecl::write_field(language_backend, out, &self.ty, &name, config);
@@ -805,7 +942,9 @@ impl Constant {
                 language_backend.write_literal(out, value);
             }
             Language::Cython => {
-                out.write("const ");
+                if !write_field_prepends_const(&self.ty) {
+                    out.write("const ");
+                }
                 // For extern Cython declarations the initializer is ignored,
                 // but still useful as documentation, so we write it as a comment.
                 crate::bindgen::cdecl::write_field(language_backend, out, &self.ty, &name, config);

tests/expectations-symbols/const_byte_string.c.sym

@@ -0,0 +1,3 @@
+{
+root;
+};

tests/expectations-symbols/const_str.c.sym

@@ -0,0 +1,3 @@
+{
+root;
+};

tests/expectations-symbols/const_str_no_static.c.sym

@@ -0,0 +1,3 @@
+{
+root;
+};

tests/expectations/assoc_constant.c

@@ -7,6 +7,7 @@ typedef struct {
 
 } Foo;
 #define Foo_GA 10
+#define Foo_BU "hello world"
 #define Foo_ZO 3.14
 
 void root(Foo x);

tests/expectations/assoc_constant.compat.c

@@ -7,6 +7,7 @@ typedef struct {
 
 } Foo;
 #define Foo_GA 10
+#define Foo_BU "hello world"
 #define Foo_ZO 3.14
 
 #ifdef __cplusplus

tests/expectations/assoc_constant.cpp

@@ -8,6 +8,7 @@ struct Foo {
 
 };
 constexpr static const int32_t Foo_GA = 10;
+constexpr static const char Foo_BU[] = "hello world";
 constexpr static const float Foo_ZO = 3.14;
 
 extern "C" {

tests/expectations/assoc_constant.pyx

@@ -9,6 +9,7 @@ cdef extern from *:
   ctypedef struct Foo:
     pass
   const int32_t Foo_GA # = 10
+  const char Foo_BU[] # = "hello world"
   const float Foo_ZO # = 3.14
 
   void root(Foo x);

tests/expectations/assoc_constant_both.c

@@ -7,6 +7,7 @@ typedef struct Foo {
 
 } Foo;
 #define Foo_GA 10
+#define Foo_BU "hello world"
 #define Foo_ZO 3.14
 
 void root(struct Foo x);