Avoid too many requests

Author: thewilkybarkid

src/Config.ts

@@ -1,4 +1,4 @@
-import { Config, Layer } from 'effect'
+import { Config, Effect, Layer, RateLimiter, pipe } from 'effect'
 import { OrcidConfig } from './Orcid.js'
 import { RedisConfig } from './Redis.js'
 import { ZenodoConfig } from './Zenodo.js'
@@ -17,7 +17,7 @@ const redisConfig: Config.Config<RedisConfig> = Config.nested(
   'REDIS',
 )
 
-const zenodoConfig: Config.Config<ZenodoConfig> = Config.nested(
+const zenodoConfig: Config.Config<Omit<ZenodoConfig, 'rateLimit'>> = Config.nested(
   Config.all({
     url: Config.mapAttempt(Config.string('URL'), url => new URL(url)),
   }),
@@ -27,5 +27,13 @@ const zenodoConfig: Config.Config<ZenodoConfig> = Config.nested(
 export const ConfigLive = Layer.mergeAll(
   Layer.effect(OrcidConfig, orcidConfig),
   Layer.effect(RedisConfig, redisConfig),
-  Layer.effect(ZenodoConfig, zenodoConfig),
+  Layer.effect(
+    ZenodoConfig,
+    pipe(
+      zenodoConfig,
+      Effect.bind('rateLimit', () =>
+        RateLimiter.make({ limit: 1, interval: '1.5 seconds', algorithm: 'fixed-window' }),
+      ),
+    ),
+  ),
 )

src/Program.ts

@@ -193,6 +193,7 @@ const processUser = (user: Users.User) =>
     Effect.annotateLogs('orcidId', user.orcidId),
     Effect.provideService(Orcid.OrcidAccessToken, { token: user.accessToken }),
     Effect.scoped,
+    Effect.delay('1 second'),
   )
 
 export const program = Users.getUsers.pipe(Stream.runForEach(processUser))

src/Zenodo.ts

@@ -1,6 +1,6 @@
 import { HttpClient } from '@effect/platform'
 import { type ParseResult, Schema } from '@effect/schema'
-import { Context, Effect, type Record } from 'effect'
+import { Context, Effect, type RateLimiter, type Record } from 'effect'
 import { DoiSchema } from './Doi.js'
 import type * as OrcidId from './OrcidId.js'
 import * as Temporal from './Temporal.js'
@@ -12,6 +12,7 @@ type GetRecordsForOrcidIdError = HttpClient.error.HttpClientError | ParseResult.
 
 export interface ZenodoConfig {
   readonly url: URL
+  readonly rateLimit: RateLimiter.RateLimiter
 }
 
 export const ZenodoConfig = Context.GenericTag<ZenodoConfig>('ZenodoConfig')
@@ -78,5 +79,6 @@ const zenodoClient = Effect.gen(function* (_) {
     HttpClient.client.filterStatusOk,
     HttpClient.client.mapRequest(HttpClient.request.acceptJson),
     HttpClient.client.mapRequest(HttpClient.request.prependUrl(new URL('/api/', config.url).href)),
+    HttpClient.client.transform(config.rateLimit),
   )
 })

src/index.ts

@@ -44,6 +44,6 @@ const ProgramLive = Layer.mergeAll(HttpClientLive, RedisLive).pipe(
   ),
 )
 
-const runnable = Effect.provide(program, ProgramLive)
+const runnable = Effect.provide(program, ProgramLive).pipe(Effect.scoped)
 
 await Effect.runPromise(runnable)