Merge pull request #747 from rm3l/add_fake_crawlers_for_testing

Author: rm3l

backend/api/src/main/kotlin/org/rm3l/devfeed/actuator/DevFeedHealthChecker.kt

@@ -33,7 +33,7 @@ import org.springframework.stereotype.Service
 class DevFeedHealthChecker(private val dao: DevFeedDao) : HealthIndicator {
 
   override fun health(): Health =
-      if (dao.getRecentArticles(limit = 1).isNotEmpty()) {
+      if (dao.getArticles(limit = 1).isNotEmpty()) {
         Health.up().build()
       } else {
         Health.down().build()

backend/api/src/main/kotlin/org/rm3l/devfeed/config/DevFeedCrawlersConfiguration.kt

@@ -25,6 +25,7 @@
 package org.rm3l.devfeed.config
 
 import java.util.concurrent.ExecutorService
+import org.rm3l.devfeed.crawlers.DummyCrawler
 import org.rm3l.devfeed.crawlers.discoverdev_io.DiscoverDevIoCrawler
 import org.rm3l.devfeed.crawlers.engineeringblogs_xyz.EngineeringBlogsCrawler
 import org.rm3l.devfeed.crawlers.rm3l_org.Rm3lOrgCrawler
@@ -41,6 +42,11 @@ class DevFeedCrawlersConfiguration {
   @Qualifier("devFeedExecutorService")
   private lateinit var devFeedExecutorService: ExecutorService
 
+  @Bean
+  @ConditionalOnProperty(
+      name = ["crawlers.dummy.enabled"], havingValue = "true", matchIfMissing = false)
+  fun dummyCrawler() = DummyCrawler()
+
   @Bean
   @ConditionalOnProperty(
       name = ["crawlers.discoverdev_io.enabled"], havingValue = "true", matchIfMissing = true)

backend/api/src/main/kotlin/org/rm3l/devfeed/crawlers/DevFeedFetcherService.kt

@@ -33,6 +33,7 @@ import javax.annotation.PostConstruct
 import javax.annotation.PreDestroy
 import org.apache.commons.lang3.concurrent.BasicThreadFactory
 import org.rm3l.devfeed.common.articleparser.ArticleExtractor
+import org.rm3l.devfeed.common.contract.Article
 import org.rm3l.devfeed.common.screenshot.ArticleScreenshotExtractor
 import org.rm3l.devfeed.crawlers.common.DevFeedCrawler
 import org.rm3l.devfeed.persistence.ArticleUpdater
@@ -181,3 +182,39 @@ class DevFeedFetcherService(
     }
   }
 }
+
+class DummyCrawler : DevFeedCrawler() {
+
+  private companion object {
+    private const val SOURCE = "https://dev-feed.example.com"
+  }
+
+  override fun call(): Collection<Article> {
+    val now = System.currentTimeMillis()
+    return listOf(
+        Article(
+            timestamp = now,
+            title = "Article Title 1",
+            description = "Lorem Ipsum Dolor Sit Amet",
+            source = SOURCE,
+            url = "$SOURCE/article1",
+            tags = listOf("tag1", "tag2"),
+        ),
+        Article(
+            timestamp = now,
+            title = "Article Title 2",
+            description = "consectetur adipiscing elit",
+            source = SOURCE,
+            url = "$SOURCE/article2",
+            tags = listOf("tag1", "tag3", "tag4"),
+        ),
+        Article(
+            timestamp = now,
+            title = "Article Title 3",
+            description = "sed do eiusmod tempor incididunt ut labore et dolore magna aliqua",
+            source = SOURCE,
+            url = "$SOURCE/article3",
+            tags = listOf("tag10", "tag3", "tag1"),
+        ))
+  }
+}

backend/api/src/main/resources/service.properties

@@ -87,6 +87,8 @@ crawlers.task.cron-expression=0 0 * * * *
 crawlers.discoverdev_io.enabled=true
 crawlers.engineeringblogx_xyz.enabled=true
 crawlers.rm3l_org.enabled=true
+# Fake article crawler, useful for testing.
+crawlers.dummy.enabled=false
 
 logging.level.org.rm3l.devfeed=INFO
 

backend/persistence/src/main/kotlin/org/rm3l/devfeed/persistence/impl/rdbms/DevFeedRdbmsDao.kt

@@ -329,7 +329,7 @@ class DevFeedRdbmsDao(
             it[timestamp] = article.timestamp
             it[title] = article.title
             it[description] =
-                if (article.description?.length ?: 0 > 10_000)
+                if ((article.description?.length ?: 0) > 10_000)
                     article.description?.substring(0 until 10_000)?.plus("...")
                 else article.description
             it[link] = article.url