Merge pull request #1139 from ia0/stable

ebursztein · web-flow · commit f44dde1d6167 · 2025-09-11T11:34:37.000-04:00
Commit the CLI to stability
diff --git a/README.md b/README.md
@@ -88,139 +88,7 @@ docker run -it --rm -v $(pwd):/magika magika -r /magika/tests_data/basic
 
 ### Command line client
 
-Magika is available as a command line tool, written in Rust (the `magika` python package ships this client). Here below are a few examples on how to use it, and the output of `magika --help`, which documents the list of options. See [here](./rust/cli/README.md) for developer notes and documentation on how to install the Rust client via `cargo`.
-
-Examples:
-
-```shell
-$ cd tests_data/basic && magika -r *
-asm/code.asm: Assembly (code)
-batch/simple.bat: DOS batch file (code)
-c/code.c: C source (code)
-css/code.css: CSS source (code)
-csv/magika_test.csv: CSV document (code)
-dockerfile/Dockerfile: Dockerfile (code)
-docx/doc.docx: Microsoft Word 2007+ document (document)
-epub/doc.epub: EPUB document (document)
-epub/magika_test.epub: EPUB document (document)
-flac/test.flac: FLAC audio bitstream data (audio)
-handlebars/example.handlebars: Handlebars source (code)
-html/doc.html: HTML document (code)
-ini/doc.ini: INI configuration file (text)
-javascript/code.js: JavaScript source (code)
-jinja/example.j2: Jinja template (code)
-jpeg/magika_test.jpg: JPEG image data (image)
-json/doc.json: JSON document (code)
-latex/sample.tex: LaTeX document (text)
-makefile/simple.Makefile: Makefile source (code)
-markdown/README.md: Markdown document (text)
-[...]
-```
-
-```shell
-$ magika ./tests_data/basic/python/code.py --json
-[
-  {
-    "path": "./tests_data/basic/python/code.py",
-    "result": {
-      "status": "ok",
-      "value": {
-        "dl": {
-          "description": "Python source",
-          "extensions": [
-            "py",
-            "pyi"
-          ],
-          "group": "code",
-          "is_text": true,
-          "label": "python",
-          "mime_type": "text/x-python"
-        },
-        "output": {
-          "description": "Python source",
-          "extensions": [
-            "py",
-            "pyi"
-          ],
-          "group": "code",
-          "is_text": true,
-          "label": "python",
-          "mime_type": "text/x-python"
-        },
-        "score": 0.753000020980835
-      }
-    }
-  }
-]
-```
-
-```shell
-$ cat doc.ini | magika -
--: INI configuration file (text)
-```
-
-```help
-$ magika --help
-Determines the content type of files with deep-learning
-
-Usage: magika [OPTIONS] [PATH]...
-
-Arguments:
-  [PATH]...
-          List of paths to the files to analyze.
-
-          Use a dash (-) to read from standard input (can only be used once).
-
-Options:
-  -r, --recursive
-          Identifies files within directories instead of identifying the directory itself
-
-      --no-dereference
-          Identifies symbolic links as is instead of identifying their content by following them
-
-      --colors
-          Prints with colors regardless of terminal support
-
-      --no-colors
-          Prints without colors regardless of terminal support
-
-  -s, --output-score
-          Prints the prediction score in addition to the content type
-
-  -i, --mime-type
-          Prints the MIME type instead of the content type description
-
-  -l, --label
-          Prints a simple label instead of the content type description
-
-      --json
-          Prints in JSON format
-
-      --jsonl
-          Prints in JSONL format
-
-      --format <CUSTOM>
-          Prints using a custom format (use --help for details).
-
-          The following placeholders are supported:
-
-            %p  The file path
-            %l  The unique label identifying the content type
-            %d  The description of the content type
-            %g  The group of the content type
-            %m  The MIME type of the content type
-            %e  Possible file extensions for the content type
-            %s  The score of the content type for the file
-            %S  The score of the content type for the file in percent
-            %b  The model output if overruled (empty otherwise)
-            %%  A literal %
-
-  -h, --help
-          Print help (see a summary with '-h')
-
-  -V, --version
-          Print version
-```
+Magika is available as a command line tool, written in Rust (the `magika` python package ships this client). The output of `magika --help` and the [README](./rust/cli/README.md) document the list of options.
 
 ### Python module and other bindings
 
diff --git a/rust/cli/CHANGELOG.md b/rust/cli/CHANGELOG.md
@@ -1,5 +1,12 @@
 # Changelog
 
+## 1.0.0-dev
+
+### Patch
+
+- Change description
+- Update dependencies
+
 ## 0.1.4
 
 ### Minor
diff --git a/rust/cli/Cargo.lock b/rust/cli/Cargo.lock
diff --git a/rust/cli/Cargo.toml b/rust/cli/Cargo.toml
@@ -1,10 +1,10 @@
 [package]
 name = "magika"
-version = "0.1.4"
+version = "1.0.0-dev"
 authors = ["Magika Developers <magika-dev@google.com>"]
 license = "Apache-2.0"
 edition = "2021"
-description = "Determines the content type of a file with deep-learning"
+description = "Determines file content types using AI"
 repository = "https://github.com/google/magika"
 homepage = "https://google.github.io/magika/"
 keywords = ["cli", "file", "magic"]
@@ -16,7 +16,7 @@ anyhow = "1.0.86"
 async-channel = "2.3.1"
 clap = { version = "4.5.9", features = ["cargo", "derive", "string"] }
 colored = "3.0.0"
-magika_lib = { package = "magika", version = "=0.2.1", path = "../lib", features = ["serde"] }
+magika_lib = { package = "magika", version = "=0.2.2-dev", path = "../lib", features = ["serde"] }
 num_cpus = "1.16.0"
 ort = "=2.0.0-rc.10"
 serde = { version = "1.0.204", features = ["derive"] }
diff --git a/rust/cli/README.md b/rust/cli/README.md
@@ -1,7 +1,7 @@
 # Magika CLI
 
 This binary crate implements a command-line interface (CLI) to the library crate
-[magika](https://crates.io/crates/magika) which provides file type detection with deep-learning.
+[magika](https://crates.io/crates/magika) which provides file content type detection using AI.
 
 ## Disclaimer
 
@@ -46,32 +46,21 @@ cargo install --locked --path=rust/cli
 ## Examples
 
 ```shell
-$ cd tests_data/basic && magika -r *
+% cd tests_data/basic && magika -r * | head
 asm/code.asm: Assembly (code)
 batch/simple.bat: DOS batch file (code)
 c/code.c: C source (code)
 css/code.css: CSS source (code)
 csv/magika_test.csv: CSV document (code)
 dockerfile/Dockerfile: Dockerfile (code)
 docx/doc.docx: Microsoft Word 2007+ document (document)
-epub/doc.epub: EPUB document (document)
-epub/magika_test.epub: EPUB document (document)
-flac/test.flac: FLAC audio bitstream data (audio)
-handlebars/example.handlebars: Handlebars source (code)
-html/doc.html: HTML document (code)
-ini/doc.ini: INI configuration file (text)
-javascript/code.js: JavaScript source (code)
-jinja/example.j2: Jinja template (code)
-jpeg/magika_test.jpg: JPEG image data (image)
-json/doc.json: JSON document (code)
-latex/sample.tex: LaTeX document (text)
-makefile/simple.Makefile: Makefile source (code)
-markdown/README.md: Markdown document (text)
-[...]
+docx/magika_test.docx: Microsoft Word 2007+ document (document)
+eml/sample.eml: RFC 822 mail (text)
+empty/empty_file: Empty file (inode)
 ```
 
 ```shell
-$ magika ./tests_data/basic/python/code.py --json
+% magika ./tests_data/basic/python/code.py --json
 [
   {
     "path": "./tests_data/basic/python/code.py",
@@ -100,21 +89,21 @@ $ magika ./tests_data/basic/python/code.py --json
           "label": "python",
           "mime_type": "text/x-python"
         },
-        "score": 0.753000020980835
+        "score": 0.996999979019165
       }
     }
   }
 ]
 ```
 
 ```shell
-$ cat doc.ini | magika -
+% cat tests_data/basic/ini/doc.ini | magika -
 -: INI configuration file (text)
 ```
 
-```help
-$ magika --help
-Determines the content type of files with deep-learning
+```shell
+% magika --help
+Determines file content types using AI
 
 Usage: magika [OPTIONS] [PATH]...
 
diff --git a/rust/cli/src/main.rs b/rust/cli/src/main.rs
@@ -31,7 +31,7 @@ use serde::Serialize;
 use tokio::fs::File;
 use tokio::io::AsyncReadExt;
 
-/// Determines the content type of files with deep-learning.
+/// Determines file content types using AI.
 #[derive(Parser)]
 #[command(name = "magika", version = Version, arg_required_else_help = true)]
 struct Flags {
diff --git a/rust/lib/CHANGELOG.md b/rust/lib/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.2.2-dev
+
+### Patch
+
+- Change description and documentation
+
 ## 0.2.1
 
 ### Patch
diff --git a/rust/lib/Cargo.lock b/rust/lib/Cargo.lock
diff --git a/rust/lib/Cargo.toml b/rust/lib/Cargo.toml
@@ -1,10 +1,10 @@
 [package]
 name = "magika"
-version = "0.2.1"
+version = "0.2.2-dev"
 authors = ["Magika Developers <magika-dev@google.com>"]
 license = "Apache-2.0"
 edition = "2021"
-description = "Determines the content type of a file with deep-learning"
+description = "Determines file content types using AI"
 repository = "https://github.com/google/magika"
 homepage = "https://google.github.io/magika/"
 keywords = ["file", "magic"]
diff --git a/rust/lib/README.md b/rust/lib/README.md
@@ -1,7 +1,7 @@
 # Magika
 
-This library crate provides file type detection with deep-learning. A command-line interface (CLI)
-for this library is provided by the [magika-cli](https://crates.io/crates/magika-cli) binary crate.
+This library crate provides file content type detection using AI. A command-line interface (CLI) for
+this library is provided by the [magika-cli](https://crates.io/crates/magika-cli) binary crate.
 
 ## Disclaimer
 
diff --git a/rust/lib/src/file.rs b/rust/lib/src/file.rs
@@ -29,14 +29,14 @@ pub enum FileType {
     /// The file is a symbolic link.
     Symlink,
 
-    /// The file is a regular file and was identified with deep-learning.
+    /// The file is a regular file and was identified using AI.
     Inferred(InferredType),
 
-    /// The file is a regular file and was identified without deep-learning.
+    /// The file is a regular file and was identified using rules.
     Ruled(ContentType),
 }
 
-/// Content type identified with deep-learning.
+/// Content type identified using AI.
 #[derive(Debug, Clone)]
 pub struct InferredType {
     /// The content type.
diff --git a/rust/lib/src/input.rs b/rust/lib/src/input.rs
@@ -21,7 +21,7 @@ use crate::config::ModelConfig;
 use crate::future::exec;
 use crate::{ContentType, Result};
 
-/// Features to identify a file with deep-learning.
+/// Features to identify a file using AI.
 pub struct Features(pub(crate) Vec<i32>);
 
 /// Synchronous abstraction over file content.
@@ -106,24 +106,24 @@ impl AsyncInputApi for tokio::fs::File {
 
 /// Result of features extraction.
 pub enum FeaturesOrRuled {
-    /// Features extracted for deep-learning.
+    /// Features extracted for identification using AI.
     Features(Features),
 
-    /// Content identified without deep-learning.
+    /// Content identified with rules.
     Ruled(ContentType),
 }
 
 impl FeaturesOrRuled {
     /// Extracts the features from a file (synchronously).
     ///
-    /// Returns the content type directly if the file is not suited for deep-learning.
+    /// Returns the content type directly if the file cannot be identified using AI.
     pub fn extract_sync(file: impl SyncInput) -> Result<Self> {
         exec(Self::extract(file))
     }
 
     /// Extracts the features from a file (asynchronously).
     ///
-    /// Returns the content type directly if the file is not suited for deep-learning.
+    /// Returns the content type directly if the file cannot be identified using AI.
     pub async fn extract_async(file: impl AsyncInput) -> Result<Self> {
         Self::extract(file).await
     }
diff --git a/rust/lib/src/lib.rs b/rust/lib/src/lib.rs
diff --git a/rust/sync.sh b/rust/sync.sh
