Server telemetry (#850)

Author: lbwexler

grails-app/services/io/xh/toolbox/SlackAlertService.groovy

@@ -14,6 +14,8 @@ import static io.xh.hoist.monitor.MonitorStatus.*
 
 class SlackAlertService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.slack'
+
     ConfigService configService
 
     void init() {
@@ -63,7 +65,7 @@ Time: ${tl.dateCreated.format('dd-MMM-yyyy HH:mm:ss')}
     }
 
     private void sendSlackMessage(message) {
-        span('toolbox.slack.sendMessage').run {
+        span('sendMessage').run {
             def client = new JSONClient(),
                 post = new HttpPost('https://slack.com/api/chat.postMessage'),
                 body = JSONSerializer.serialize([channel: config.channel, text: message]),

grails-app/services/io/xh/toolbox/app/GitHubService.groovy

@@ -33,6 +33,8 @@ import static io.xh.hoist.util.DateTimeUtils.MINUTES
  */
 class GitHubService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.github'
+
     static clearCachesConfigs = ['gitHubRepos', 'gitHubAccessToken', 'gitHubMaxPagesPerLoad']
 
     ConfigService configService
@@ -87,7 +89,7 @@ class GitHubService extends BaseService {
         def repos = configService.getList('gitHubRepos', []),
             newCommitCount = 0
 
-        span('toolbox.github.getCommits')
+        span('getCommits')
             .logInfo("Refreshing GitHub commits for ${repos.size()} configured repositories")
             .run {
                 repos.each {

grails-app/services/io/xh/toolbox/app/NewsService.groovy

@@ -13,6 +13,8 @@ import static io.xh.hoist.util.DateTimeUtils.getMINUTES
 
 class NewsService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.news'
+
     static clearCachesConfigs = ['newsSources', 'newsApiKey']
 
     ConfigService configService
@@ -43,7 +45,7 @@ class NewsService extends BaseService {
     // Implementation
     //------------------------
     private List<NewsItem> loadNews() {
-        span('toolbox.news.getNews').run {
+        span('getNews').run {
             def sources = configService.getMap('newsSources').keySet().toList(),
                 sourcesParam = sources.join(','),
                 apiKey = configService.getString('newsApiKey'),

grails-app/services/io/xh/toolbox/app/RecallsService.groovy

@@ -8,13 +8,15 @@ import static org.apache.hc.core5.http.HttpStatus.SC_OK
 
 class RecallsService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.recalls'
+
     def configService
 
     Integer lastResponseCode
     private JSONClient _jsonClient
 
     List fetchRecalls(String searchQuery) {
-        span('toolbox.recalls.get').run {
+        span('get').run {
             def host = configService.getString('recallsHost'),
                 uri = !searchQuery ?
                     // `_exists_:openfda` ensures all search hits includes a nested openfda object that contains essential data for frontend

grails-app/services/io/xh/toolbox/app/WeatherService.groovy

@@ -11,6 +11,8 @@ import static io.xh.hoist.util.DateTimeUtils.MINUTES
 
 class WeatherService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.weather'
+
     static clearCachesConfigs = ['weatherApiKey']
 
     ConfigService configService
@@ -38,7 +40,7 @@ class WeatherService extends BaseService {
             encodedCity = URLEncoder.encode(city, 'UTF-8'),
             url = "https://api.openweathermap.org/data/2.5/weather?q=${encodedCity}&appid=${apiKey}&units=imperial"
 
-        span('toolbox.weather.getCurrent')
+        span('getCurrent')
             .logDebug("Loading forecast for $city")
             .run {
                 client.executeAsMap(new HttpGet(url))
@@ -50,7 +52,7 @@ class WeatherService extends BaseService {
             encodedCity = URLEncoder.encode(city, 'UTF-8'),
             url = "https://api.openweathermap.org/data/2.5/forecast?q=${encodedCity}&appid=${apiKey}&units=imperial"
 
-        span('toolbox.weather.getForecast')
+        span('getForecast')
             .logDebug("Loading forecast for $city")
             .run {
                 client.executeAsMap(new HttpGet(url))

grails-app/services/io/xh/toolbox/portfolio/PortfolioService.groovy

@@ -1,8 +1,6 @@
 package io.xh.toolbox.portfolio
 
 import com.hazelcast.replicatedmap.ReplicatedMap
-import io.micrometer.core.instrument.Gauge
-import io.micrometer.core.instrument.Timer
 import io.xh.hoist.BaseService
 import io.xh.hoist.cachedvalue.CachedValue
 import io.xh.hoist.exception.DataNotAvailableException
@@ -12,7 +10,6 @@ import java.time.*
 
 import static io.xh.toolbox.portfolio.Utils.*
 import static io.xh.hoist.util.DateTimeUtils.SECONDS
-import static java.util.concurrent.TimeUnit.MILLISECONDS
 
 
 /**
@@ -26,17 +23,17 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS
  */
 class PortfolioService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.portfolio'
+
+    MetricsService metricsService
+
     def configService,
         orderGenerationService,
         historicalPriceGenerationService,
         instrumentGenerationService
 
-    MetricsService metricsService
-
     private CachedValue<Portfolio> _portfolio = createCachedValue(name: 'portfolio', replicate: true)
     private ReplicatedMap<String, MarketPrice> _currentPrices = createReplicatedMap('currentPrices')
-    private Timer generationTimer
-    private Gauge positionsGauge
 
     void init() {
         initMetrics()
@@ -79,16 +76,24 @@ class PortfolioService extends BaseService {
     // Implementation
     //------------------------
     private void initMetrics() {
-        def registry = metricsService.registry
+        metricsService.registerGauge(
+            name: 'position.count',
+            description: 'Number of portfolio positions',
+            valueFn: { _portfolio.get()?.rawPositions?.size() ?: 0 },
+            owner: this
+        )
 
-        positionsGauge = Gauge.builder('toolbox.portfolio.positions', this) {
-            (_portfolio.get()?.rawPositions?.size() ?: 0) as double
-        }.description('Number of portfolio positions')
-            .register(registry)
+        metricsService.configureTimer(
+            name: 'generation.time',
+            description: 'Time to generate portfolio',
+            owner: this
+        )
 
-        generationTimer = Timer.builder('toolbox.portfolio.generationTime')
-            .description('Time to generate portfolio')
-            .register(registry)
+        metricsService.configureCounter(
+            name: 'generation.count',
+            description: 'The number of portfolio generations',
+            owner: this
+        )
     }
 
     private void updateData(boolean force = false) {
@@ -126,9 +131,10 @@ class PortfolioService extends BaseService {
 
     private Portfolio generatePortfolio() {
         observe()
-            .span(name: 'generatePortfolio')
+            .span('generatePortfolio')
+            .timer('generation.time')
+            .counter('generation.count')
             .logInfo('Generating Portfolio')
-            .timer(generationTimer)
             .run {
                 def day = LocalDate.now(),
                     instruments = instrumentGenerationService.generateInstruments(),
@@ -183,8 +189,6 @@ class PortfolioService extends BaseService {
         def p = _portfolio.get()
         return [
             config: configForAdminStats('portfolioConfigs'),
-            avgGenerationTime: generationTimer.mean(MILLISECONDS),
-
             portfolioAvailable: portfolioAvailable,
             day: p?.day,
             generatedAt: p?.timeCreated,

grails-app/services/io/xh/toolbox/portfolio/generation/HistoricalPriceGenerationService.groovy

@@ -17,6 +17,8 @@ import static io.xh.toolbox.portfolio.Utils.randInt
  */
 class HistoricalPriceGenerationService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.portfolio'
+
     def tradingDayService
 
     /** Generate a daily price series for each instrument, keyed by symbol. */
@@ -25,7 +27,7 @@ class HistoricalPriceGenerationService extends BaseService {
             LocalDate day
     ) {
         observe()
-            .span(name: 'generateHistoricalPrices')
+            .span('generateHistoricalPrices')
             .logDebug("Generating historical prices for ${instruments.size()} instruments")
             .run {
                 List<LocalDate> tradingDays = tradingDayService.historicalDays(day)

grails-app/services/io/xh/toolbox/portfolio/generation/InstrumentGenerationService.groovy

@@ -15,12 +15,14 @@ import static io.xh.toolbox.portfolio.Utils.sample
  */
 class InstrumentGenerationService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.portfolio'
+
     def configService
 
     /** Generate a map of unique instruments keyed by ticker symbol. */
     Map<String, Instrument> generateInstruments() {
         observe()
-            .span(name: 'generateInstruments')
+            .span('generateInstruments')
             .logDebug("Generating ${config.instrumentCount} instruments")
             .run {
                 def instrumentCount = config.instrumentCount

grails-app/services/io/xh/toolbox/portfolio/generation/OrderGenerationService.groovy

@@ -22,6 +22,8 @@ import static io.xh.toolbox.portfolio.Lookups.*
  */
 class OrderGenerationService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.portfolio'
+
     def configService
 
     private static final ZoneOffset EST = ZoneOffset.ofHours(-5)
@@ -33,7 +35,7 @@ class OrderGenerationService extends BaseService {
             Map<String, List<MarketPrice>> historicalPrices
     ) {
         observe()
-            .span(name: 'generateOrders')
+            .span('generateOrders')
             .logDebug("Generating ${config.orderCount} orders")
             .run {
                 def orderCount = config.orderCount

grails-app/services/io/xh/toolbox/security/AuthZeroTokenService.groovy

@@ -16,6 +16,8 @@ import static java.lang.System.currentTimeMillis
  */
 class AuthZeroTokenService extends BaseService {
 
+    String telemetryPrefix = 'toolbox.auth'
+
     static clearCachesConfigs = ['auth0Config']
 
     AuthenticationService authenticationService
@@ -91,7 +93,7 @@ class AuthZeroTokenService extends BaseService {
 
     private JsonWebKeySet createKeySet() {
         def url = "https://$domain/.well-known/jwks.json"
-        span('toolbox.auth.getAuth0JWKS')
+        span('getAuth0JWKS')
             .logInfo(['Fetching JWKS', url])
             .run {
                 def jwksJson = (new JSONClient()).executeAsString(new HttpGet(url)),