Merge pull request #590 from SierraSoftworks/copilot/migrate-src-filters-module

Migrate filter DSL to the filt-rs crate

Author: notheotherben

Cargo.lock

@@ -10,6 +10,7 @@ chrono = { version = "0.4.45", features = ["serde"] }
 clap = { version = "4.6.1", features = ["derive", "string"] }
 croner = "3.0.1"
 ctrlc = "3.5.2"
+filt-rs = { version = "1.1.0", features = ["chrono", "regex", "serde"] }
 gix = { version = "0.84.0", features = [
   "blocking-http-transport-reqwest-rust-tls",
 ] }

Cargo.toml

@@ -19,6 +19,9 @@ Here are a few common filter examples which you might use in your configuration.
 - `!release.prerelease && !asset.source-code` - Only include release artifacts which are not marked as pre-releases and are not source code archives.
 - `repo.name in ["git-tool", "grey"]` - Only include repositories with the names "git-tool" or "grey".
 - `repo.stargazers >= 5` - Only include repositories with at least 5 stars.
+- `repo.name like "*-backup"` - Only include repositories whose name ends with "-backup" using glob pattern matching.
+- `repo.name matches r"^awesome-\d+$"` - Only include repositories whose name matches the given regular expression.
+- `repo.pushed_at > now() - 30d` - Only include repositories which have been pushed to within the last 30 days.
 
 ## Language Features
 ### Properties - `repo.<field>`
@@ -46,6 +49,12 @@ If you wish to treat an empty string as a valid value, you can use `repo.<field>
 evaluation of an empty string.
 :::
 
+::: tip
+You can also write *raw strings* using an `r` prefix (for example `r"^v\d+$"`), within which backslashes are treated literally
+rather than as escape sequences. This is particularly convenient when writing [regular expression](#pattern-matching-like-matches)
+patterns. Use the hashed form `r#"..."#` if your pattern needs to contain a double quote.
+:::
+
 #### Numbers
 Numbers are represented internally as a 64-bit floating-point value, which means that they can represent most reasonably sized
 integers as well as most reasonably precise decimal numbers. For example, `5` and `5.0` are equivalent in the filter language.
@@ -61,6 +70,18 @@ example, `repo.fork` will evaluate to `true` if the repository is a fork, and `f
 The `null` value is used to represent the absence of a value, and is considered falsey when evaluated. Accessing a property which
 does not exist will return `null`.
 
+#### Datetimes and Durations
+Some fields, such as `repo.pushed_at` or `release.published_at`, expose native timestamps rather than strings. These can be compared
+against one another, and against the current time using the [`now()`](#functions) function, allowing you to backup only those entities
+which have changed recently.
+
+Durations are written as a number immediately followed by a unit (`ms`, `s`, `m` for minutes, `h`, `d`, or `w`), and several segments
+can be chained together to form a more precise duration, for example `1h30m`. Datetimes and durations support `+` and `-` arithmetic,
+so `now() - 7d` evaluates to the point in time seven days ago.
+
+ - `repo.pushed_at > now() - 30d` - Only include repositories which have been pushed to within the last 30 days.
+ - `release.published_at < now() - 1w` - Only include releases which were published more than a week ago.
+
 ## Operators
 ### Unary Negation - `!`
 The unary negation operator converts the following expression into the boolean opposite of its value.
@@ -102,8 +123,9 @@ comparison. These operators **DO NOT** perform type coercion, which means that y
 type - for example, comparing `5 <= "5" || 5 >= "5"` will always return `false`.
 
 ::: warning
-String comparisons are performed using a case-insensitive comparison of ASCII characters, which means that `"Hello" == "hello"` will return `true`,
-as will `"hello👋" == "hello"`.
+String comparisons are performed case-insensitively using the filter language's Unicode case-folding rules, which means that
+`"Hello" == "hello"` will return `true`, as will `"STRASSE" == "straße"`. If you need an exact, case-sensitive comparison, use the
+[`_cs` variants](#case-sensitivity-cs) of the string operators.
 :::
 
  - `==` - Returns `true` if the left and right hand expressions are equal.
@@ -136,6 +158,35 @@ The prefix and suffix matching operators are used to determine whether a string
  - `"hello" startswith "he"` - Determines whether the string `hello` starts with the sequence `he`, returning `true` in this case.
  - `"goodbye" endswith "bye"` - Determines whether the string `goodbye` ends with the sequence `bye`, returning `true` in this case.
 
+### Pattern Matching - `like`, `matches`
+The pattern matching operators allow you to match a string against a pattern, which can be useful when you want to match
+repositories whose names follow a particular convention without listing each of them explicitly.
+
+ - `like` performs a case-insensitive [glob](https://en.wikipedia.org/wiki/Glob_(programming)) match, where `*` matches any
+   sequence of characters (including none), `?` matches exactly one character, and a backslash makes the following character
+   literal (`\*`, `\?`, `\\`). For example, `repo.name like "*-rs"` matches any repository whose name ends with `-rs`.
+ - `matches` performs a [regular expression](https://docs.rs/regex/latest/regex/#syntax) match. Regular expressions are
+   case-sensitive (use `(?i)` to ignore case) and unanchored (use `^` and `$` to anchor the match). For example,
+  `release.tag matches r"^v\d+(\.\d+){2}$"` matches tags like `v1.2.3`.
+
+::: tip
+Regular expression patterns are easiest to write using [raw strings](#strings) (`r"..."`), which do not process backslash
+escape sequences and so avoid the need to double-escape characters like `\d`.
+:::
+
+### Case Sensitivity - `_cs`
+The string operators (`contains`, `in`, `startswith`, `endswith`, and `like`) compare values case-insensitively by default. Each of
+them has a case-sensitive variant with a `_cs` suffix (`contains_cs`, `in_cs`, `startswith_cs`, `endswith_cs`, and `like_cs`) which
+compares strings exactly as written. The `matches` operator is always case-sensitive unless you opt in with the `(?i)` flag.
+
+## Functions
+Filters may call built-in functions using the familiar `name(args...)` syntax. Unknown function names and incorrect argument counts
+are rejected when the filter is parsed.
+
+ - `now()` - Returns the current UTC time, evaluated afresh on every evaluation. This is most useful in combination with
+   [durations](#datetimes-and-durations), for example `repo.pushed_at > now() - 30d`.
+ - `trim(string)` - Returns the string argument with leading and trailing whitespace removed (`null` for non-string values).
+
 ## Nerdy Details
 The filtering language itself is implemented as a simple recursive descent parser which compiles an expression
 tree from the input string. This expression tree is then evaluated using an interpreter to determine whether

docs/advanced/filters.md

@@ -65,3 +65,5 @@ These fields are also available when using [`github/repo`](./repo.md) or [`githu
 | `gist.file_names`       | `array`   | List of file names in the gist                 |
 | `gist.languages`        | `array`   | List of programming languages used in the gist |
 | `gist.type`             | `string`  | MIME-Type of content in the gist               |
+| `gist.created_at`       | `datetime`| When the gist was created                      |
+| `gist.updated_at`       | `datetime`| When the gist was last updated                 |

docs/reference/gist.md

@@ -70,9 +70,13 @@ For `kind: github/release`
 | `release.draft`       | `boolean`  | Whether the release is a draft (unpublished) release               |
 | `release.prerelease`  | `boolean`  | Whether to identify the release as a prerelease or a full release  |
 | `release.published`   | `boolean`  | Whether the release is a published (not a draft) release           |
+| `release.created_at`  | `datetime` | When the release was created (_2013-02-27T19:35:32Z_)              |
+| `release.published_at`| `datetime` | When the release was published, or `null` for drafts (_2013-02-27T19:35:32Z_) |
 | `asset.name`          | `string`   | The file name of the asset (_github-backup-darwin-arm64_)          |
 | `asset.size`          | `integer`  | The size of the asset, in kilobytes. (_1024_)                      |
 | `asset.downloaded`    | `boolean`  | If the asset was downloaded at least once from the GitHub Release  |
+| `asset.created_at`    | `datetime` | When the asset was created (_2013-02-27T19:35:32Z_)               |
+| `asset.updated_at`    | `datetime` | When the asset was last updated (_2013-02-27T19:35:32Z_)          |
 
 ```json
 {
@@ -117,7 +121,11 @@ For `kind: github/release`
     // Whether the release is a draft (inverse of published)
     "draft": false,
     /// Whether the release has been published yet or not (inverse of draft)
-    "published": true
+    "published": true,
+    // When the release was created
+    "created_at": "2013-02-27T19:35:32Z",
+    // When the release was published (null for draft releases)
+    "published_at": "2013-02-27T19:35:32Z"
   },
 
   // Describes a specific artifact which is part of a release
@@ -127,7 +135,11 @@ For `kind: github/release`
     // The size of the release asset in kilobytes
     "size": 1024,
     // Whether the asset has been downloaded at least once
-    "downloaded": true
+    "downloaded": true,
+    // When the asset was created
+    "created_at": "2013-02-27T19:35:32Z",
+    // When the asset was last updated
+    "updated_at": "2013-02-27T19:35:32Z"
   }
 }
 ```

docs/reference/release.md

@@ -73,6 +73,9 @@ These fields are also available when using [`github/release`](./release.md) or [
 | `repo.template`        | `boolean`  | Whether this repository acts as a template that can be used to generate new repositories           |
 | `repo.forks`           | `integer`  | The number of times this repository is forked                                                      |
 | `repo.stargazers`      | `integer`  | The number of people starred this repository                                                       |
+| `repo.pushed_at`       | `datetime` | When the repository was last pushed to (_2011-01-26T19:06:43Z_)                                     |
+| `repo.created_at`      | `datetime` | When the repository was created (_2011-01-26T19:01:12Z_)                                            |
+| `repo.updated_at`      | `datetime` | When the repository was last updated (_2011-01-26T19:14:43Z_)                                       |
 
 ```json
 {
@@ -102,7 +105,13 @@ These fields are also available when using [`github/release`](./release.md) or [
     // The number of times this repository has been forked.
     "forks": 0,
     // The number of people who have starred this repository.
-    "stargazers": 501
+    "stargazers": 501,
+    // When the repository was last pushed to.
+    "pushed_at": "2011-01-26T19:06:43Z",
+    // When the repository was created.
+    "created_at": "2011-01-26T19:01:12Z",
+    // When the repository was last updated.
+    "updated_at": "2011-01-26T19:14:43Z"
   }
 }
 ```

docs/reference/repo.md

@@ -29,7 +29,7 @@ macro_rules! entity {
             }
             )*
 
-            pub fn with_metadata<V: Into<FilterValue>>(mut self, key: &'static str, value: V) -> Self {
+            pub fn with_metadata<'a, V: Into<FilterValue<'a>>>(mut self, key: &'static str, value: V) -> Self {
                 self.metadata.insert(key, value.into());
                 self
             }
@@ -47,7 +47,7 @@ macro_rules! entity {
         }
 
         impl crate::Filterable for $name {
-            fn get(&self, key: &str) -> crate::FilterValue {
+            fn get(&self, key: &str) -> crate::FilterValue<'_> {
                 self.metadata.get(key)
             }
         }
@@ -79,7 +79,7 @@ mod tests {
         assert_eq!(entity.url, "http://example.com");
         assert_eq!(entity.credentials, Credentials::Token("test".to_string()));
 
-        assert_eq!(entity.get("test"), FilterValue::String("test".to_string()));
+        assert_eq!(entity.get("test"), FilterValue::String("test".into()));
         assert_eq!(entity.get("test2"), FilterValue::Number(1_f64));
     }
 }

src/entities/macros.rs

@@ -7,6 +7,7 @@ use crate::{FilterValue, Filterable};
 
 pub use credentials::Credentials;
 pub use release::Release;
+use std::borrow::Cow;
 use std::collections::HashMap;
 use unicase::UniCase;
 
@@ -18,21 +19,36 @@ pub trait BackupEntity: std::fmt::Display + Filterable {
 }
 
 #[derive(Default, Clone, Debug)]
-pub struct Metadata(HashMap<UniCase<&'static str>, FilterValue>);
+pub struct Metadata(HashMap<UniCase<&'static str>, FilterValue<'static>>);
 
 impl Metadata {
-    pub fn insert<V: Into<FilterValue>>(&mut self, key: &'static str, value: V) {
-        self.0.insert(UniCase::new(key), value.into());
+    pub fn insert<'a, V: Into<FilterValue<'a>>>(&mut self, key: &'static str, value: V) {
+        self.0.insert(UniCase::new(key), into_owned(value.into()));
     }
 
-    pub fn get(&self, key: &str) -> FilterValue {
+    pub fn get(&self, key: &str) -> FilterValue<'_> {
         self.0
             .get(&UniCase::new(key))
             .cloned()
             .unwrap_or(FilterValue::Null)
     }
 }
 
+/// Converts a [`FilterValue`] into one which owns all of its data so that it
+/// can be cached within a [`Metadata`] collection (whose entries must outlive
+/// the entity they were derived from).
+fn into_owned(value: FilterValue<'_>) -> FilterValue<'static> {
+    match value {
+        FilterValue::Null => FilterValue::Null,
+        FilterValue::Bool(b) => FilterValue::Bool(b),
+        FilterValue::Number(n) => FilterValue::Number(n),
+        FilterValue::String(s) => FilterValue::String(Cow::Owned(s.into_owned())),
+        FilterValue::Tuple(v) => FilterValue::Tuple(v.into_iter().map(into_owned).collect()),
+        FilterValue::DateTime(dt) => FilterValue::DateTime(dt),
+        FilterValue::Duration(d) => FilterValue::Duration(d),
+    }
+}
+
 pub trait MetadataSource {
     fn inject_metadata(&self, metadata: &mut Metadata);
 }