feat(contentlayer): use provideSchema as Stream

examples/node-script-notion/package.json

@@ -2,7 +2,8 @@
   "name": "node-script-notion-example",
   "private": true,
   "scripts": {
-    "start": "contentlayer build && node --experimental-json-modules my-script.mjs"
+    "start": "contentlayer build && node --experimental-json-modules my-script.mjs",
+    "dev": "contentlayer dev"
   },
   "dependencies": {
     "@notionhq/client": "^2.2.3",

packages/@contentlayer/cli/src/commands/PostInstallCommand.ts

@@ -1,8 +1,9 @@
 import * as path from 'node:path'
+import { config } from 'node:process'
 
 import * as core from '@contentlayer/core'
 import { fs } from '@contentlayer/utils'
-import { OT, pipe, T } from '@contentlayer/utils/effect'
+import { O, OT, pipe, S, T } from '@contentlayer/utils/effect'
 
 import { BaseCommand } from './_BaseCommand.js'
 
@@ -60,7 +61,7 @@ const generateTypes = ({
     }
 
     const { source, esbuildHash } = sourceEither.right
-    const schemaDef = yield* $(source.provideSchema(esbuildHash))
+    const schemaDef = yield* $(pipe(source.provideSchema(esbuildHash), S.runHead, T.map(O.getUnsafe)))
 
     if (!indexDtsFileExists) {
       const options = source.options

packages/@contentlayer/core/src/dynamic-build.ts

@@ -99,7 +99,7 @@ export const dynamicBuild = ({ config, verbose }: { config: Config; verbose: boo
   pipe(
     T.gen(function* ($) {
       // TODO try to do this as generation-time in the future and come up with a serialization strategy for the `schemaDef`
-      const schemaDef = yield* $(config.source.provideSchema(config.esbuildHash))
+      const schemaDef = yield* $(pipe(config.source.provideSchema(config.esbuildHash), S.runHead, T.map(O.getUnsafe)))
 
       const cache = yield* $(
         pipe(

packages/@contentlayer/core/src/generation/generate-dotpkg.ts

@@ -1,7 +1,7 @@
 import type { AbsolutePosixFilePath, RelativePosixFilePath } from '@contentlayer/utils'
 import { filePathJoin, fs, relative } from '@contentlayer/utils'
 import type { E, HasClock, HasConsole } from '@contentlayer/utils/effect'
-import { Array, Chunk, flow, OT, pipe, S, T } from '@contentlayer/utils/effect'
+import { Array, Chunk, flow, OT, pipe, S, T, Tp } from '@contentlayer/utils/effect'
 import type { GetContentlayerVersionError } from '@contentlayer/utils/node'
 import { getContentlayerVersion } from '@contentlayer/utils/node'
 import { camelCase } from 'camel-case'
@@ -82,20 +82,26 @@ export const generateDotpkgStream = ({
 > => {
   const writtenFilesCache = {}
   const generationOptions = { sourcePluginType: config.source.type, options: config.source.options }
-  const resolveParams = pipe(
-    T.structPar({
-      schemaDef: config.source.provideSchema(config.esbuildHash),
-      targetPath: ArtifactsDir.mkdir,
-    }),
-    T.either,
+
+  const provideSchema = pipe(
+    config.source.provideSchema(config.esbuildHash),
+    S.mapEffect((schemaDef) =>
+      pipe(
+        T.structPar({
+          schemaDef: T.succeed(schemaDef),
+          targetPath: ArtifactsDir.mkdir,
+        }),
+      ),
+    ),
+    S.either,
   )
 
   // .pipe(
   //   tap((artifactsDir) => watchData && errorIfArtifactsDirIsDeleted({ artifactsDir }))
   // ),
 
   return pipe(
-    S.fromEffect(resolveParams),
+    provideSchema,
     S.chainMapEitherRight(({ schemaDef, targetPath }) =>
       pipe(
         config.source.fetchData({ schemaDef, verbose }),

packages/@contentlayer/core/src/plugin.ts

@@ -121,7 +121,7 @@ export type SourcePlugin = {
 
 export type ProvideSchema = (
   esbuildHash: string,
-) => T.Effect<OT.HasTracer & HasConsole, SourceProvideSchemaError, SchemaDef>
+) => S.Stream<OT.HasTracer & HasClock & HasConsole, SourceProvideSchemaError, SchemaDef>
 export type FetchData = (_: {
   schemaDef: SchemaDef
   verbose: boolean

packages/@contentlayer/experimental-source-files-stackbit/src/__test__/stackbit.spec.ts

@@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url'
 import * as SourceFiles from '@contentlayer/source-files'
 import { provideDummyTracing } from '@contentlayer/utils'
 import type { HasClock, HasConsole, OT } from '@contentlayer/utils/effect'
-import { pipe, provideTestConsole, T } from '@contentlayer/utils/effect'
+import { O, pipe, provideTestConsole, S, T } from '@contentlayer/utils/effect'
 import * as Stackbit from '@stackbit/sdk'
 import { expect, test } from 'vitest'
 
@@ -36,7 +36,7 @@ const schemaFromFixture = (fixtureName: string) => async () => {
   const documentTypes = stackbitConfigToDocumentTypes(configResult.config!)
 
   const contentlayerSource = await SourceFiles.makeSource({ contentDirPath: NOT_USED_STR, documentTypes })(undefined)
-  const schema = await runMain(contentlayerSource.provideSchema(NOT_USED_STR))
+  const schema = await runMain(pipe(contentlayerSource.provideSchema(NOT_USED_STR), S.runHead, T.map(O.getUnsafe)))
 
   return schema.result
 }

packages/@contentlayer/source-contentful/src/index.ts

@@ -34,7 +34,7 @@ export const makeSourcePlugin: core.MakeSourcePlugin<Args & PluginOptions> = (ar
     type: 'contentful',
     extensions,
     options,
-    provideSchema: () => provideSchema({ accessToken, spaceId, environmentId, options, schemaOverrides }),
+    provideSchema: () => S.fromEffect(provideSchema({ accessToken, spaceId, environmentId, options, schemaOverrides })),
     fetchData: ({ schemaDef }) =>
       pipe(
         S.fromEffect(

packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts

@@ -8,7 +8,7 @@ import {
   unknownToRelativePosixFilePath,
 } from '@contentlayer/utils'
 import type { HasClock, HasConsole, OT } from '@contentlayer/utils/effect'
-import { pipe, provideTestConsole, T, These } from '@contentlayer/utils/effect'
+import { O, pipe, provideTestConsole, S, T, These } from '@contentlayer/utils/effect'
 import { NodeFsLive } from '@contentlayer/utils/node'
 
 import type { HasDocumentTypeMapState } from '../../fetchData/DocumentTypeMap.js'
@@ -33,7 +33,7 @@ export const runTest = async ({
     const esbuildHash = 'not-important-for-this-test'
 
     const source = yield* $(T.tryPromise(() => makeSource({ contentDirPath, documentTypes })(undefined)))
-    const coreSchemaDef = yield* $(source.provideSchema(esbuildHash))
+    const coreSchemaDef = yield* $(pipe(source.provideSchema(esbuildHash), S.runHead, T.map(O.getUnsafe)))
 
     const documentTypeDefs = (Array.isArray(documentTypes) ? documentTypes : Object.values(documentTypes)).map((_) =>
       _.def(),

packages/@contentlayer/source-files/src/__test__/type-generation/basic.spec.ts

@@ -1,6 +1,6 @@
 import { renderTypes } from '@contentlayer/core'
 import { provideJaegerTracing } from '@contentlayer/utils'
-import { pipe, provideConsole, T } from '@contentlayer/utils/effect'
+import { O, pipe, provideConsole, S, T } from '@contentlayer/utils/effect'
 import { expect, test } from 'vitest'
 
 import type { DocumentTypes } from '../../index.js'
@@ -11,7 +11,7 @@ const renderTypeSource = async (documentTypes: DocumentTypes) => {
   const esbuildHash = 'not-important-for-this-test'
   const schemaDef = await pipe(
     T.tryPromise(() => makeSource({ documentTypes, contentDirPath: '' })(undefined)),
-    T.chain((source) => source.provideSchema(esbuildHash)),
+    T.chain((source) => pipe(source.provideSchema(esbuildHash), S.runHead, T.map(O.getUnsafe))),
     provideJaegerTracing('contentlayer-cli'),
     provideConsole,
     T.runPromise,

packages/@contentlayer/source-files/src/index.ts

@@ -99,9 +99,11 @@ export const makeSource: core.MakeSourcePlugin<Args> = (args) => async (sourceKe
     extensions: extensions ?? {},
     options,
     provideSchema: (esbuildHash) =>
-      pipe(
-        makeCoreSchema({ documentTypeDefs, options, esbuildHash }),
-        T.mapError((error) => new SourceProvideSchemaError({ error })),
+      S.fromEffect(
+        pipe(
+          makeCoreSchema({ documentTypeDefs, options, esbuildHash }),
+          T.mapError((error) => new SourceProvideSchemaError({ error })),
+        ),
       ),
     fetchData: ({ schemaDef, verbose, skipCachePersistence }) =>
       pipe(

packages/@contentlayer/source-notion/package.json

@@ -1,6 +1,6 @@
 {
   "name": "contentlayer-source-notion",
-  "version": "0.0.1-alpha.23",
+  "version": "0.0.1-alpha.24",
   "type": "module",
   "exports": "./dist/index.js",
   "types": "./dist/index.d.ts",

packages/@contentlayer/source-notion/src/index.ts

@@ -1,6 +1,6 @@
 import type * as core from '@contentlayer/core'
 import { processArgs } from '@contentlayer/core'
-import { pipe, S, T } from '@contentlayer/utils/effect'
+import { pipe, S, SC, T } from '@contentlayer/utils/effect'
 import { NotionRenderer } from '@notion-render/client'
 import * as notion from '@notionhq/client'
 
@@ -41,9 +41,14 @@ export const makeSource: core.MakeSourcePlugin<PluginOptions & core.PartialArgs>
     options,
     provideSchema: () =>
       pipe(
-        provideSchema({ databaseTypeDefs, options }),
-        T.provideService(NotionClient)(client),
-        T.provideService(NotionRendererTag)(renderer),
+        S.fromEffect(
+          pipe(
+            provideSchema({ databaseTypeDefs, options }),
+            T.provideService(NotionClient)(client),
+            T.provideService(NotionRendererTag)(renderer),
+          ),
+        ),
+        S.repeatSchedule(SC.spaced(5_000)),
       ),
     fetchData: ({ schemaDef }) =>
       pipe(

packages/@contentlayer/source-notion/src/schema/provideSchema.ts

@@ -14,6 +14,7 @@ export type ProvideSchemaArgs = {
 export const provideSchema = ({ databaseTypeDefs, options }: ProvideSchemaArgs) =>
   pipe(
     T.gen(function* ($) {
+      console.log('Providing schema')
       return {
         documentTypeDefMap: yield* $(provideDocumentTypeDefMap({ databaseTypeDefs, options })),
         nestedTypeDefMap: yield* $(provideNestedTypeDefMap()),
@@ -29,7 +30,10 @@ export const provideSchema = ({ databaseTypeDefs, options }: ProvideSchemaArgs)
             hash,
           }),
         ),
-        T.tap((schemaDef) => T.succeed(core.validateSchema(schemaDef))),
+        T.tap((schemaDef) => {
+          console.log(schemaDef.hash)
+          return T.succeed(core.validateSchema(schemaDef))
+        }),
       ),
     ),
 

packages/@contentlayer/source-notion/src/types.ts

@@ -8,6 +8,9 @@ export type PluginOptions = {
   client?: ConstructorParameters<typeof notion.Client>[0] | notion.Client
   renderer?: ConstructorParameters<typeof NotionRenderer>[0] | NotionRenderer
   databaseTypes: DatabaseTypes
+  dev?: {
+    polling: false | number
+  }
 }
 
 export type FieldDef = core.FieldDef & { propertyKey?: string }

packages/contentlayer-stackbit-yaml-generator/src/__test__/fixtures/index.ts

@@ -1,5 +1,5 @@
 import { provideJaegerTracing } from '@contentlayer/utils'
-import { pipe, provideConsole, T } from '@contentlayer/utils/effect'
+import { O, pipe, provideConsole, S, T } from '@contentlayer/utils/effect'
 import type { DocumentType } from 'contentlayer/source-files'
 import { makeSource } from 'contentlayer/source-files'
 
@@ -14,7 +14,7 @@ const esbuildHash = 'not-important-for-this-test'
 const makeSchema = (documentTypes: Record<string, DocumentType<any>>) =>
   pipe(
     T.tryPromise(() => makeSource({ documentTypes, contentDirPath: '' })(undefined)),
-    T.chain((source) => source.provideSchema(esbuildHash)),
+    T.chain((source) => pipe(source.provideSchema(esbuildHash), S.runHead, T.map(O.getUnsafe))),
     provideJaegerTracing('contentlayer-cli'),
     provideConsole,
     T.runPromise,

packages/contentlayer-stackbit-yaml-generator/src/cli/DefaultCommand.ts

@@ -5,8 +5,8 @@ import type { HasCwd } from '@contentlayer/core'
 import { getConfig, provideCwd } from '@contentlayer/core'
 import type { fs } from '@contentlayer/utils'
 import { provideJaegerTracing, recRemoveUndefinedValues } from '@contentlayer/utils'
-import type { HasConsole } from '@contentlayer/utils/effect'
-import { OT, pipe, pretty, provideConsole, T } from '@contentlayer/utils/effect'
+import type { HasClock, HasConsole } from '@contentlayer/utils/effect'
+import { O, OT, pipe, pretty, provideConsole, S, T } from '@contentlayer/utils/effect'
 import { NodeFsLive } from '@contentlayer/utils/node'
 import { Command, Option } from 'clipanion'
 import * as t from 'typanion'
@@ -53,11 +53,14 @@ export class DefaultCommand extends Command {
     }
   }
 
-  executeSafe = (): T.Effect<OT.HasTracer & HasCwd & HasConsole & fs.HasFs, unknown, void> =>
+  executeSafe = (): T.Effect<OT.HasTracer & HasCwd & HasConsole & HasClock & fs.HasFs, unknown, void> =>
     pipe(
       getConfig({ configPath: this.configPath }),
       T.chain((config) =>
-        T.struct({ source: T.succeed(config.source), schema: config.source.provideSchema(config.esbuildHash) }),
+        T.struct({
+          source: T.succeed(config.source),
+          schema: pipe(config.source.provideSchema(config.esbuildHash), S.runHead, T.map(O.getUnsafe)),
+        }),
       ),
       T.chain(({ schema, source }) =>
         T.tryCatchPromise(