Normalize question/exclamation marks

Fix #34

Author: dahlia

.vscode/settings.json

@@ -24,5 +24,9 @@
   "deno.enable": false,
   "deno.enablePaths": [
     "scripts/deno/"
-  ]
+  ],
+
+  "haskell.serverEnvironment": {
+    "STACK_YAML": "stack-ghc-9.4.yaml"
+  }
 }

CHANGES.md

@@ -6,11 +6,21 @@ Version 0.5.0
 
 To be released.
 
+ -  Stops normalizer now normalizes question marks and exclamation marks too.
+    The below functions were added.  [[#34]]
+
+    Haskell API-wise, the below record fields were added to
+    the `Text.Seonbi.Punctuation.Stops` data type:
+
+     -  `questionMark` field
+     -  `exclamationMark` field
+
  -  Added prebuilt executable binaries for Linux ARM64 (linux-arm64).
 
  -  The official Docker images are now multi-platform (linux/amd64 and
     linux/arm64).  [[#35]]
 
+[#34]: https://github.com/dahlia/seonbi/issues/34
 [#35]: https://github.com/dahlia/seonbi/issues/35
 
 

en.utf-8.add

@@ -1,7 +1,12 @@
 deno
+guillemets
 hanja
+inequal
+interpunct
+interpuncts
 phoneticize
 phoneticized
+punct
 seonbi
 Sino
 stdict

src/Text/Seonbi/Punctuation.hs

@@ -249,49 +249,51 @@ data TitlePunct
     deriving (Eq, Show)
 
 
--- | A set of stops—'period', 'comma', and 'interpunct'—to be used by
--- 'normalizeStops' function.
+-- | A set of stops—'period', 'comma', 'interpunct', 'questionMark', and
+-- 'exclamationMark'—to be used by 'normalizeStops' function.
 --
 -- There are three presets: 'horizontalStops', 'verticalStops', and
 -- 'horizontalStopsWithSlashes'.
 data Stops = Stops
     { period :: Text
     , comma :: Text
     , interpunct :: Text
+    , questionMark :: Text
+    , exclamationMark :: Text
     } deriving (Eq, Show)
 
 -- | Stop sentences in the modern Korean style which follows Western stops.
 -- E.g.:
 --
--- > 봄·여름·가을·겨울. 어제, 오늘.
+-- > 봄·여름·가을·겨울. 어제, 오늘. 새벽? 아침!
 horizontalStops :: Stops
 horizontalStops = Stops
     { period = ". "
     , comma = ", "
     , interpunct = "·"
+    , questionMark = "? "
+    , exclamationMark = "! "
     }
 
 -- | Stop sentences in the pre-modern Korean style which follows Chinese stops.
 -- E.g.:
 --
--- > 봄·여름·가을·겨울。어제、오늘。
+-- > 봄·여름·가을·겨울。어제、오늘。새벽？아침！
 verticalStops :: Stops
 verticalStops = Stops
     { period = "。"
     , comma = "、"
     , interpunct = "·"
+    , questionMark = "？"
+    , exclamationMark = "！"
     }
 
 -- | Similar to 'horizontalStops' except slashes are used instead of
 -- interpuncts. E.g.:
 --
--- > 봄/여름/가을/겨울. 어제, 오늘.
+-- > 봄/여름/가을/겨울. 어제, 오늘. 새벽? 아침!
 horizontalStopsWithSlashes :: Stops
-horizontalStopsWithSlashes = Stops
-    { period = ". "
-    , comma = ", "
-    , interpunct = "/"
-    }
+horizontalStopsWithSlashes = horizontalStops { interpunct = "/" }
 
 
 -- | Normalizes sentence stops (periods, commas, and interpuncts).
@@ -332,6 +334,12 @@ normalizeStops stops input = (`fmap` annotatedEntities) $ \ case
         , do { ending <- interpunct'
              ; return (toEntity $ adjustEnding ending $ interpunct stops)
              }
+        , do { ending <- questionMark'
+             ; return (toEntity $ adjustEnding ending $ questionMark stops)
+             }
+        , do { ending <- exclamationMark'
+             ; return (toEntity $ adjustEnding ending $ exclamationMark stops)
+             }
         ]
     adjustEnding :: Ending -> Text -> Text
     adjustEnding ending text
@@ -377,6 +385,26 @@ normalizeStops stops input = (`fmap` annotatedEntities) $ \ case
         , string "&#183;"
         , asciiCI "&#xb7;"
         ] >> return Ending
+    questionMark' :: Parser Ending
+    questionMark' = choice
+        [ char '?' >> boundary
+        , char '？' >> trailingSpaces
+        , string "&quest;" >> boundary
+        , string "&#63;" >> boundary
+        , asciiCI "&#x3f;" >> boundary
+        , string "&#65311;" >> trailingSpaces
+        , asciiCI "&#xff1f;" >> trailingSpaces
+        ]
+    exclamationMark' :: Parser Ending
+    exclamationMark' = choice
+        [ char '!' >> boundary
+        , char '！' >> trailingSpaces
+        , string "&excl;" >> boundary
+        , string "&#33;" >> boundary
+        , asciiCI "&#x21;" >> boundary
+        , string "&#65281;" >> trailingSpaces
+        , asciiCI "&#xff01;" >> trailingSpaces
+        ]
     closingChars :: String
     closingChars =
         [ '"', '”', '\'', '’', ')', ']', '}', '」', '』', '〉', '》', '）', '〕'
@@ -419,7 +447,7 @@ normalizeStops stops input = (`fmap` annotatedEntities) $ \ case
         ]
 
 
-data Ending = TrailingChars Text | TrailingSpaces Text | Ending
+data Ending = TrailingChars Text | TrailingSpaces Text | Ending deriving (Show)
 
 
 -- | Substitution options for 'transformArrow' function.  These options can

test/Text/Seonbi/PunctuationSpec.hs

@@ -475,15 +475,25 @@ spec = do
                 [ "·", "·", "·", "·"
                 , "·", "·", "·"
                 ] :: [Text]
+        let questionMarks =
+                [ "? ", "? ", "? ", "? "
+                , "？", "？", "？"
+                ] :: [Text]
+        let exclamationMarks =
+                [ "! ", "! ", "! ", "! "
+                , "！", "！", "！"
+                ] :: [Text]
         let s = stripEnd
         let examples =
-                [ [qc|봄{i1}여름{i2}가을{i3}겨울{p1}(括弧{s p3}) 어제{c}오늘{s p2}|]
+                [ [qc|봄{i1}여름{i2}가을{i3}겨울{p1}(括弧{s p3}) 어제{c}오늘{p2}새벽{q}아침{s e}|]
                 | p1 <- periods, p2 <- periods, p3 <- periods
                 , c <- commas
                 , i1 <- interpuncts, i2 <- interpuncts, i3 <- interpuncts
+                , q <- questionMarks
+                , e <- exclamationMarks
                 ] :: [Text]
         let gen = unsafePerformIO getStdGen :: StdGen
-        let randomInts = randomRs (0, 499) gen :: [Int]
+        let randomInts = randomRs (0, 999) gen :: [Int]
         let sampledExamples =
                 [ e
                 | (e, r) <- Prelude.zip examples randomInts
@@ -499,19 +509,19 @@ spec = do
                 normalizeStops horizontalStops input `shouldBe`
                     [ HtmlStartTag [] P ""
                     , HtmlText [P]
-                        "봄&#xb7;여름&#xb7;가을&#xb7;겨울. (括弧.) 어제, 오늘."
+                        "봄&#xb7;여름&#xb7;가을&#xb7;겨울. (括弧.) 어제, 오늘. 새벽? 아침!"
                     , HtmlEndTag [] P
                     ]
                 normalizeStops verticalStops input `shouldBe`
                     [ HtmlStartTag [] P ""
                     , HtmlText [P]
                         ("봄&#xb7;여름&#xb7;가을&#xb7;겨울&#x3002;(括弧&#x3002;) " <>
-                            "어제&#x3001;오늘&#x3002;")
+                            "어제&#x3001;오늘&#x3002;새벽&#xff1f;아침&#xff01;")
                     , HtmlEndTag [] P
                     ]
                 normalizeStops horizontalStopsWithSlashes input `shouldBe`
                     [ HtmlStartTag [] P ""
-                    , HtmlText [P] "봄/여름/가을/겨울. (括弧.) 어제, 오늘."
+                    , HtmlText [P] "봄/여름/가을/겨울. (括弧.) 어제, 오늘. 새벽? 아침!"
                     , HtmlEndTag [] P
                     ]
         it "normalizes stops followed by boundaries as well" $ do