diff --git a/Package.swift b/Package.swift index f2705fef1..e5201a88f 100644 --- a/Package.swift +++ b/Package.swift @@ -68,6 +68,7 @@ let package = Package( .product(name: "OpenAPIKitCompat", package: "OpenAPIKit"), .product(name: "Algorithms", package: "swift-algorithms"), .product(name: "OrderedCollections", package: "swift-collections"), + .product(name: "HeapModule", package: "swift-collections"), .product(name: "Yams", package: "Yams"), ], swiftSettings: swiftSettings diff --git a/Sources/_OpenAPIGeneratorCore/Config.swift b/Sources/_OpenAPIGeneratorCore/Config.swift index 74bb6f13e..c30145563 100644 --- a/Sources/_OpenAPIGeneratorCore/Config.swift +++ b/Sources/_OpenAPIGeneratorCore/Config.swift @@ -26,6 +26,85 @@ public enum NamingStrategy: String, Sendable, Codable, Equatable, CaseIterable { case idiomatic } +/// Configuration for sharding the generated Types file into multiple files +/// organized by dependency layers. +public struct ShardingConfig: Sendable, Equatable { + /// The number of type shards per layer (index 0 = component/leaf layer, + /// index 1+ = dependent layers). + public var typeShardCounts: [Int] + /// Maximum Swift files per shard for type schemas. + public var maxFilesPerShard: Int + /// Maximum Swift files per shard for operations. + public var maxFilesPerShardOps: Int + /// The number of shards per operation layer (index = layer number). + /// Must have exactly `layerCount` entries. + public var operationLayerShardCounts: [Int] + /// Module prefix for deterministic file naming (e.g., "MyServiceAPI"). + /// When set, produces deterministic file names suitable for build-system integration. + public var modulePrefix: String? + + /// The total number of dependency layers. + public var layerCount: Int { typeShardCounts.count } + + /// Returns the number of type shards for the given layer index. + public func typeShardCount(forLayer layerIndex: Int) -> Int { + typeShardCounts[layerIndex] + } + + public init( + typeShardCounts: [Int], + maxFilesPerShard: Int = 25, + maxFilesPerShardOps: Int = 16, + operationLayerShardCounts: [Int], + modulePrefix: String? = nil + ) { + self.typeShardCounts = typeShardCounts + self.maxFilesPerShard = maxFilesPerShard + self.maxFilesPerShardOps = maxFilesPerShardOps + self.operationLayerShardCounts = operationLayerShardCounts + self.modulePrefix = modulePrefix + } + + public enum ValidationError: Error, CustomStringConvertible { + case nonPositiveShardCount(field: String, value: Int) + case shardCountMismatch(field: String, expected: Int, got: Int) + + public var description: String { + switch self { + case .nonPositiveShardCount(let field, let value): + return "\(field) must be > 0, got \(value)" + case .shardCountMismatch(let field, let expected, let got): + return "\(field) count (\(got)) must equal layerCount (\(expected))" + } + } + } + + public func validate() throws { + func requirePositive(_ value: Int, field: String) throws { + guard value > 0 else { + throw ValidationError.nonPositiveShardCount(field: field, value: value) + } + } + for (index, count) in typeShardCounts.enumerated() { + try requirePositive(count, field: "typeShardCounts[\(index)]") + } + try requirePositive(maxFilesPerShard, field: "maxFilesPerShard") + try requirePositive(maxFilesPerShardOps, field: "maxFilesPerShardOps") + for (index, count) in operationLayerShardCounts.enumerated() { + try requirePositive(count, field: "operationLayerShardCounts[\(index)]") + } + if operationLayerShardCounts.count != layerCount { + throw ValidationError.shardCountMismatch( + field: "operationLayerShardCounts", + expected: layerCount, + got: operationLayerShardCounts.count + ) + } + } +} + +extension ShardingConfig: Codable {} + /// A structure that contains configuration options for a single execution /// of the generator pipeline run. /// @@ -68,6 +147,9 @@ public struct Config: Sendable { /// Additional pre-release features to enable. public var featureFlags: FeatureFlags + /// Optional sharding configuration for splitting Types output into multiple files. + public var sharding: ShardingConfig? + /// Creates a configuration with the specified generator mode and imports. /// - Parameters: /// - mode: The mode to use for generation. @@ -90,7 +172,8 @@ public struct Config: Sendable { namingStrategy: NamingStrategy, nameOverrides: [String: String] = [:], typeOverrides: TypeOverrides = .init(), - featureFlags: FeatureFlags = [] + featureFlags: FeatureFlags = [], + sharding: ShardingConfig? = nil ) { self.mode = mode self.access = access @@ -101,5 +184,6 @@ public struct Config: Sendable { self.nameOverrides = nameOverrides self.typeOverrides = typeOverrides self.featureFlags = featureFlags + self.sharding = sharding } } diff --git a/Sources/_OpenAPIGeneratorCore/GeneratorPipeline.swift b/Sources/_OpenAPIGeneratorCore/GeneratorPipeline.swift index 8f4a8cf13..4e03cb6a7 100644 --- a/Sources/_OpenAPIGeneratorCore/GeneratorPipeline.swift +++ b/Sources/_OpenAPIGeneratorCore/GeneratorPipeline.swift @@ -85,6 +85,24 @@ public func runGenerator(input: InMemoryInputFile, config: Config, diagnostics: -> InMemoryOutputFile { try makeGeneratorPipeline(config: config, diagnostics: diagnostics).run(input) } +/// Runs the generator and returns multiple output files when sharding is configured. +/// Falls back to a single-element array when sharding is not configured. +public func runShardedGenerator( + input: InMemoryInputFile, + config: Config, + diagnostics: any DiagnosticCollector +) throws -> [InMemoryOutputFile] { + let pipeline = makeGeneratorPipeline(config: config, diagnostics: diagnostics) + let parsed = try pipeline.parseOpenAPIFileStage.run(input) + let translated = try pipeline.translateOpenAPIToStructuredSwiftStage.run(parsed) + return translated.files.map { namedFile in + let renderer = TextBasedRenderer.default + renderer.renderFile(namedFile.contents) + let string = renderer.renderedContents() + return InMemoryOutputFile(baseName: namedFile.name, contents: Data(string.utf8)) + } +} + /// Creates a new pipeline instance. /// - Parameters: /// - parser: An OpenAPI document parser. diff --git a/Sources/_OpenAPIGeneratorCore/GraphAlgorithms.swift b/Sources/_OpenAPIGeneratorCore/GraphAlgorithms.swift new file mode 100644 index 000000000..e6b3f63a0 --- /dev/null +++ b/Sources/_OpenAPIGeneratorCore/GraphAlgorithms.swift @@ -0,0 +1,214 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftOpenAPIGenerator open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftOpenAPIGenerator project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftOpenAPIGenerator project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +import HeapModule + +enum GraphAlgorithms { + + // MARK: - Tarjan SCC (iterative for stack safety on large schemas) + + struct SCCResult: Sendable { + var componentIdOf: [String: Int] + var components: [[String]] + } + + static func tarjanSCC(graph: [String: Set]) -> SCCResult { + let sortedGraph = graph.mapValues { $0.sorted() } + var index = 0 + var sccStack: [String] = [] + var indices: [String: Int] = [:] + var lowlinks: [String: Int] = [:] + var onStack: Set = [] + var components: [[String]] = [] + + struct CallFrame { + let node: String + let neighbors: [String] + var neighborIndex: Int + } + + func strongConnect(startNode: String) { + var callStack: [CallFrame] = [] + + indices[startNode] = index + lowlinks[startNode] = index + index += 1 + sccStack.append(startNode) + onStack.insert(startNode) + + let startNeighbors = sortedGraph[startNode] ?? [] + callStack.append(CallFrame(node: startNode, neighbors: startNeighbors, neighborIndex: 0)) + + while !callStack.isEmpty { + let currentIndex = callStack.count - 1 + let frame = callStack[currentIndex] + let v = frame.node + + if frame.neighborIndex < frame.neighbors.count { + let w = frame.neighbors[frame.neighborIndex] + callStack[currentIndex].neighborIndex += 1 + + if indices[w] == nil { + indices[w] = index + lowlinks[w] = index + index += 1 + sccStack.append(w) + onStack.insert(w) + + let wNeighbors = sortedGraph[w] ?? [] + callStack.append(CallFrame(node: w, neighbors: wNeighbors, neighborIndex: 0)) + } else if onStack.contains(w) { + if let lowV = lowlinks[v], let idxW = indices[w] { + lowlinks[v] = min(lowV, idxW) + } + } + } else { + callStack.removeLast() + + if let parentIndex = callStack.indices.last { + let parent = callStack[parentIndex].node + if let lowParent = lowlinks[parent], let lowV = lowlinks[v] { + lowlinks[parent] = min(lowParent, lowV) + } + } + + if lowlinks[v] == indices[v] { + var component: [String] = [] + while true { + let w = sccStack.removeLast() + onStack.remove(w) + component.append(w) + if w == v { break } + } + components.append(component.sorted()) + } + } + } + } + + for v in graph.keys.sorted() where indices[v] == nil { + strongConnect(startNode: v) + } + + let componentIdOf = Dictionary( + uniqueKeysWithValues: components.enumerated().flatMap { compId, members in + members.map { ($0, compId) } + } + ) + + return SCCResult(componentIdOf: componentIdOf, components: components) + } + + // MARK: - Topological Sort + + static func topologicalSort(predecessors: [[Int]]) -> [Int] { + let n = predecessors.count + guard n > 0 else { return [] } + + var successors = Array(repeating: [Int](), count: n) + var inDegree = Array(repeating: 0, count: n) + for (v, preds) in predecessors.enumerated() { + inDegree[v] = preds.count + for u in preds { + successors[u].append(v) + } + } + + var heap = Heap() + for i in 0..], + scc: SCCResult + ) -> [[Int]] { + var dagPredecessors = Array(repeating: Set(), count: scc.components.count) + + for (u, neighbors) in graph { + guard let cu = scc.componentIdOf[u] else { continue } + for v in neighbors { + if let cv = scc.componentIdOf[v], cu != cv { + dagPredecessors[cu].insert(cv) + } + } + } + + return dagPredecessors.map { Array($0).sorted() } + } + + // MARK: - Longest-Path Layering + + static func longestPathLayering(dagPredecessors: [[Int]]) -> [Int] { + let topo = topologicalSort(predecessors: dagPredecessors) + var layerOf = Array(repeating: 0, count: dagPredecessors.count) + + for u in topo { + if let maxPredLayer = dagPredecessors[u].map({ layerOf[$0] }).max() { + layerOf[u] = maxPredLayer + 1 + } else { + layerOf[u] = 0 + } + } + + return layerOf + } + + // MARK: - LPT Bin-Packing + + typealias Island = [String] + + static func lptPacking( + islands: [Island], + binCount: Int, + weight: (Island) -> Int + ) -> [[Island]] { + guard binCount > 0 else { return [] } + + var bins = Array(repeating: (weight: 0, items: [Island]()), count: binCount) + + let weightedIslands = islands.map { ($0, weight($0)) } + let sortedIslands = weightedIslands.sorted { lhs, rhs in + if lhs.1 != rhs.1 { + return lhs.1 > rhs.1 + } + return lhs.0.lexicographicallyPrecedes(rhs.0) + } + + for (island, islandWeight) in sortedIslands { + let binIndex = bins.indices.min(by: { bins[$0].weight < bins[$1].weight })! + bins[binIndex].weight += islandWeight + bins[binIndex].items.append(island) + } + + return bins.map(\.items) + } +} diff --git a/Sources/_OpenAPIGeneratorCore/Layers/StructuredSwiftRepresentation.swift b/Sources/_OpenAPIGeneratorCore/Layers/StructuredSwiftRepresentation.swift index 4a7cacef5..a4ee189f8 100644 --- a/Sources/_OpenAPIGeneratorCore/Layers/StructuredSwiftRepresentation.swift +++ b/Sources/_OpenAPIGeneratorCore/Layers/StructuredSwiftRepresentation.swift @@ -27,6 +27,9 @@ struct ImportDescription: Equatable, Codable { /// For example, if there are type imports like `import Foo.Bar`, they would be listed here. var moduleTypes: [String]? + /// Whether this is an `@_exported` import (re-export). + var exported: Bool = false + /// The name of the private interface for an `@_spi` import. /// /// For example, if `spi` was "Secret" and the module name was "Foo" then the import @@ -1073,8 +1076,17 @@ struct NamedFileDescription: Equatable, Codable { /// A file with contents made up of structured Swift code. struct StructuredSwiftRepresentation: Equatable, Codable { - /// The contents of the file. - var file: NamedFileDescription + /// All output files. For non-sharded output this contains a single file. + /// For sharded output this contains the root file plus all shard files. + var files: [NamedFileDescription] + + init(file: NamedFileDescription) { + self.files = [file] + } + + init(files: [NamedFileDescription]) { + self.files = files + } } // MARK: - Conveniences diff --git a/Sources/_OpenAPIGeneratorCore/Renderer/TextBasedRenderer.swift b/Sources/_OpenAPIGeneratorCore/Renderer/TextBasedRenderer.swift index 6d9f54685..accf24d3e 100644 --- a/Sources/_OpenAPIGeneratorCore/Renderer/TextBasedRenderer.swift +++ b/Sources/_OpenAPIGeneratorCore/Renderer/TextBasedRenderer.swift @@ -92,7 +92,7 @@ struct TextBasedRenderer: RendererProtocol { func render(structured: StructuredSwiftRepresentation, config: Config, diagnostics: any DiagnosticCollector) throws -> InMemoryOutputFile { - let namedFile = structured.file + let namedFile = structured.files[0] renderFile(namedFile.contents) let string = writer.rendered() return InMemoryOutputFile(baseName: namedFile.name, contents: Data(string.utf8)) @@ -150,12 +150,13 @@ struct TextBasedRenderer: RendererProtocol { /// Renders a single import statement. func renderImport(_ description: ImportDescription) { func render(preconcurrency: Bool) { + let exportedPrefix = description.exported ? "@_exported " : "" let spiPrefix = description.spi.map { "@_spi(\($0)) " } ?? "" let preconcurrencyPrefix = preconcurrency ? "@preconcurrency " : "" if let moduleTypes = description.moduleTypes { - for type in moduleTypes { writer.writeLine("\(preconcurrencyPrefix)\(spiPrefix)import \(type)") } + for type in moduleTypes { writer.writeLine("\(preconcurrencyPrefix)\(exportedPrefix)\(spiPrefix)import \(type)") } } else { - writer.writeLine("\(preconcurrencyPrefix)\(spiPrefix)import \(description.moduleName)") + writer.writeLine("\(preconcurrencyPrefix)\(exportedPrefix)\(spiPrefix)import \(description.moduleName)") } } diff --git a/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/SchemaDependencyGraph.swift b/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/SchemaDependencyGraph.swift new file mode 100644 index 000000000..033120be2 --- /dev/null +++ b/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/SchemaDependencyGraph.swift @@ -0,0 +1,172 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftOpenAPIGenerator open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftOpenAPIGenerator project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftOpenAPIGenerator project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +import OpenAPIKit + +struct SchemaDependencyGraph { + /// Adjacency list used externally only in tests for verifying graph structure. + var edges: [String: Set] + var scc: GraphAlgorithms.SCCResult + var layerOf: [Int] + + var layerCount: Int { + (layerOf.max() ?? -1) + 1 + } + + func layer(of schema: String) -> Int? { + guard let compId = scc.componentIdOf[schema] else { return nil } + return layerOf[compId] + } + + static func build(from schemas: OpenAPI.ComponentDictionary) -> SchemaDependencyGraph { + let schemaNames = Set(schemas.map(\.key.rawValue)) + var edges: [String: Set] = [:] + + for (key, schema) in schemas { + let schemaName = key.rawValue + var dependencies = Set() + collectSchemaRefs(schema, into: &dependencies) + dependencies.remove(schemaName) + // Only keep dependencies that exist in the filtered schema set + dependencies.formIntersection(schemaNames) + edges[schemaName] = dependencies + } + + let scc = GraphAlgorithms.tarjanSCC(graph: edges) + let dagPredecessors = GraphAlgorithms.buildCondensationDAG(graph: edges, scc: scc) + let layerOf = GraphAlgorithms.longestPathLayering(dagPredecessors: dagPredecessors) + + return SchemaDependencyGraph(edges: edges, scc: scc, layerOf: layerOf) + } + + private static func resolve( + _ either: Either, T>, + in components: OpenAPI.Components + ) -> T? { + switch either { + case .a(let ref): try? components.lookup(ref) + case .b(let value): value + } + } + + static func operationSchemaRefs( + _ operation: OpenAPI.Operation, + in components: OpenAPI.Components + ) -> Set { + var refs = Set() + + if let requestBody = operation.requestBody, + let resolved = resolve(requestBody, in: components) { + for (_, content) in resolved.content { + collectContentSchemaRefs(content, into: &refs) + } + } + + for (_, responseRef) in operation.responses { + if let resolved = resolve(responseRef, in: components) { + for (_, content) in resolved.content { + collectContentSchemaRefs(content, into: &refs) + } + if let headers = resolved.headers { + for (_, headerRef) in headers { + if let header = resolve(headerRef, in: components) { + collectHeaderSchemaRefs(header, into: &refs) + } + } + } + } + } + + for paramRef in operation.parameters { + if let param = resolve(paramRef, in: components) { + switch param.schemaOrContent { + case .a(let schemaContext): + collectSchemaOrRefRefs(schemaContext.schema, into: &refs) + case .b(let contentMap): + for (_, content) in contentMap { + collectContentSchemaRefs(content, into: &refs) + } + } + } + } + + return refs + } + + private static func collectSchemaOrRefRefs( + _ schemaOrRef: Either, JSONSchema>, + into acc: inout Set + ) { + switch schemaOrRef { + case .a(let ref): + if case .internal(let internalRef) = ref.jsonReference, + case .component(name: let name) = internalRef { + acc.insert(name) + } + case .b(let jsonSchema): + collectSchemaRefs(jsonSchema, into: &acc) + } + } + + private static func collectContentSchemaRefs(_ content: OpenAPI.Content, into acc: inout Set) { + guard let schema = content.schema else { return } + collectSchemaOrRefRefs(schema, into: &acc) + } + + private static func collectHeaderSchemaRefs(_ header: OpenAPI.Header, into acc: inout Set) { + switch header.schemaOrContent { + case .a(let schemaContext): + collectSchemaOrRefRefs(schemaContext.schema, into: &acc) + case .b(let contentMap): + for (_, content) in contentMap { + collectContentSchemaRefs(content, into: &acc) + } + } + } + + private static func collectSchemaRefs(_ schema: JSONSchema, into acc: inout Set) { + switch schema.value { + case .reference(let ref, _): + if case .internal(let internalRef) = ref, case .component(name: let name) = internalRef { + acc.insert(name) + } + + case .object(_, let ctx): + for (_, prop) in ctx.properties { + collectSchemaRefs(prop, into: &acc) + } + if let additionalProps = ctx.additionalProperties { + switch additionalProps { + case .a: break + case .b(let schema): collectSchemaRefs(schema, into: &acc) + } + } + + case .array(_, let ctx): + if let items = ctx.items { + collectSchemaRefs(items, into: &acc) + } + + case .all(of: let schemas, _), .one(of: let schemas, _), .any(of: let schemas, _): + for schema in schemas { + collectSchemaRefs(schema, into: &acc) + } + + case .not(let schema, _): + collectSchemaRefs(schema, into: &acc) + + default: + break + } + } +} diff --git a/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/TypesFileTranslator.swift b/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/TypesFileTranslator.swift index ea8e6aa9a..f01862ab8 100644 --- a/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/TypesFileTranslator.swift +++ b/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/TypesFileTranslator.swift @@ -43,11 +43,27 @@ struct TypesFileTranslator: FileTranslator { let serversDecl = translateServers(doc.servers) let multipartSchemaNames = try parseSchemaNamesUsedInMultipart(paths: doc.paths, components: doc.components) - let components = try translateComponents(doc.components, multipartSchemaNames: multipartSchemaNames) let operationDescriptions = try OperationDescription.all(from: doc.paths, in: doc.components, context: context) + + if let shardingConfig = config.sharding { + return try translateFileSharded( + doc: doc, + topComment: topComment, + imports: imports, + apiProtocol: apiProtocol, + apiProtocolExtension: apiProtocolExtension, + serversDecl: serversDecl, + multipartSchemaNames: multipartSchemaNames, + operationDescriptions: operationDescriptions, + shardingConfig: shardingConfig + ) + } + let operations = try translateOperations(operationDescriptions) + let components = try translateComponents(doc.components, multipartSchemaNames: multipartSchemaNames) + let typesFile = FileDescription( topComment: topComment, imports: imports, @@ -59,4 +75,193 @@ struct TypesFileTranslator: FileTranslator { return StructuredSwiftRepresentation(file: .init(name: GeneratorMode.types.outputFileName, contents: typesFile)) } + + private func translateFileSharded( + doc: ParsedOpenAPIRepresentation, + topComment: Comment, + imports: [ImportDescription], + apiProtocol: Declaration, + apiProtocolExtension: Declaration, + serversDecl: Declaration, + multipartSchemaNames: Set, + operationDescriptions: [OperationDescription], + shardingConfig: ShardingConfig + ) throws -> StructuredSwiftRepresentation { + try shardingConfig.validate() + + let naming: ShardNamingStrategy = if let prefix = shardingConfig.modulePrefix { + .prefixed(modulePrefix: prefix) + } else { + .default + } + let importResolver = ShardImportResolver(config: shardingConfig, naming: naming) + + let shardedSchemas = try translateSchemasSharded( + doc.components.schemas, + multipartSchemaNames: multipartSchemaNames, + shardingConfig: shardingConfig, + naming: naming + ) + + let shardedOperations = try translateOperationsSharded( + operationDescriptions, + schemaGraph: shardedSchemas.graph, + maxLayer: shardedSchemas.maxLayer, + shardingConfig: shardingConfig, + naming: naming + ) + + let parameters = try translateComponentParameters(doc.components.parameters) + let requestBodies = try translateComponentRequestBodies(doc.components.requestBodies) + let responses = try translateComponentResponses(doc.components.responses) + let headers = try translateComponentHeaders(doc.components.headers) + + var allFiles: [NamedFileDescription] = [] + + allFiles.append(assembleRootFile( + topComment: topComment, + imports: imports, + exportedImports: importResolver.exportedImportsForRootFile(maxLayer: shardedSchemas.maxLayer), + apiProtocol: apiProtocol, + apiProtocolExtension: apiProtocolExtension + )) + + allFiles.append(contentsOf: assembleComponentFiles( + shardedSchemas: shardedSchemas, + naming: naming, + importResolver: importResolver, + topComment: topComment, + imports: imports, + parameters: parameters, + requestBodies: requestBodies, + responses: responses, + headers: headers + )) + + allFiles.append(contentsOf: assembleOperationFiles( + shardedOperations: shardedOperations, + naming: naming, + importResolver: importResolver, + topComment: topComment, + imports: imports, + serversDecl: serversDecl + )) + + return StructuredSwiftRepresentation(files: allFiles) + } + + private func assembleRootFile( + topComment: Comment, + imports: [ImportDescription], + exportedImports: [ImportDescription] = [], + apiProtocol: Declaration, + apiProtocolExtension: Declaration + ) -> NamedFileDescription { + NamedFileDescription( + name: "Types_root.swift", + contents: FileDescription( + topComment: topComment, + imports: imports + exportedImports, + codeBlocks: [ + .declaration(apiProtocol), + .declaration(apiProtocolExtension), + ] + ) + ) + } + + private func assembleComponentFiles( + shardedSchemas: ShardedSchemaResult, + naming: ShardNamingStrategy, + importResolver: ShardImportResolver, + topComment: Comment, + imports: [ImportDescription], + parameters: Declaration, + requestBodies: Declaration, + responses: Declaration, + headers: Declaration + ) -> [NamedFileDescription] { + let emptySchemasDecl: Declaration = .commentable( + JSONSchema.sectionComment(), + .enum(accessModifier: config.access, name: Constants.Components.Schemas.namespace, members: []) + ) + let componentsDecl: Declaration = .commentable( + .doc("Types generated from the components section of the OpenAPI document."), + .enum(.init( + accessModifier: config.access, + name: "Components", + members: [emptySchemasDecl, parameters, requestBodies, responses, headers] + )) + ) + + var files: [NamedFileDescription] = [] + + files.append(NamedFileDescription( + name: naming.componentsBaseFileName, + contents: FileDescription( + topComment: topComment, + imports: imports, + codeBlocks: [.declaration(componentsDecl)] + ) + )) + + for file in shardedSchemas.files { + let schemasExtension: Declaration = .extension( + .init(onType: "Components.Schemas", declarations: file.declarations) + ) + let isComponentLayer = file.layer == 0 + let additionalImports = isComponentLayer + ? importResolver.componentShardImports() + : importResolver.typeLayerShardImports(layerIndex: file.layer - 1) + files.append(NamedFileDescription( + name: file.fileName, + contents: FileDescription( + topComment: topComment, + imports: imports + additionalImports, + codeBlocks: file.declarations.isEmpty ? [] : [.declaration(schemasExtension)] + ) + )) + } + + return files + } + + private func assembleOperationFiles( + shardedOperations: ShardedOperationResult, + naming: ShardNamingStrategy, + importResolver: ShardImportResolver, + topComment: Comment, + imports: [ImportDescription], + serversDecl: Declaration + ) -> [NamedFileDescription] { + var files: [NamedFileDescription] = [] + + let emptyOperationsEnum: Declaration = .enum( + .init(accessModifier: config.access, name: Constants.Operations.namespace, members: []) + ) + files.append(NamedFileDescription( + name: naming.operationsBaseFileName, + contents: FileDescription( + topComment: topComment, + imports: imports, + codeBlocks: [.declaration(serversDecl), .declaration(emptyOperationsEnum)] + ) + )) + + for file in shardedOperations.files { + let operationsExtension: Declaration = .extension( + .init(onType: Constants.Operations.namespace, declarations: file.declarations) + ) + files.append(NamedFileDescription( + name: file.fileName, + contents: FileDescription( + topComment: topComment, + imports: imports + importResolver.operationShardImports(layerIndex: file.layer), + codeBlocks: file.declarations.isEmpty ? [] : [.declaration(operationsExtension)] + ) + )) + } + + return files + } } diff --git a/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/translateSchemas.swift b/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/translateSchemas.swift index 679629753..cb5c03998 100644 --- a/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/translateSchemas.swift +++ b/Sources/_OpenAPIGeneratorCore/Translator/TypesTranslator/translateSchemas.swift @@ -13,7 +13,36 @@ //===----------------------------------------------------------------------===// import OpenAPIKit +private func typeLayerSuffix(layer: Int) -> String { + "Types_L\(layer + 1)" +} + extension TypesFileTranslator { + private func declarationNodeCount(_ declaration: Declaration) -> Int { + switch declaration { + case .commentable(_, let inner): + return 1 + declarationNodeCount(inner) + case .deprecated(_, let inner): + return 1 + declarationNodeCount(inner) + case .extension(let description): + return 1 + description.declarations.map(declarationNodeCount).reduce(0, +) + case .struct(let description): + return 1 + description.members.map(declarationNodeCount).reduce(0, +) + case .enum(let description): + return 1 + description.members.map(declarationNodeCount).reduce(0, +) + case .protocol(let description): + return 1 + description.members.map(declarationNodeCount).reduce(0, +) + case .variable, + .typealias, + .function, + .enumCase: + return 1 + } + } + + private func declarationNodeCount(_ declarations: [Declaration]) -> Int { + declarations.map(declarationNodeCount).reduce(0, +) + } /// Returns a list of declarations for the provided schema, defined in the /// OpenAPI document under the specified component key. @@ -75,4 +104,332 @@ extension TypesFileTranslator { ) return componentsSchemasEnum } + + struct ShardedFile { + var layer: Int + var shardIndex: Int + var fileIndex: Int + var fileName: String + var declarations: [Declaration] + } + + struct ShardedSchemaResult { + var files: [ShardedFile] + var graph: SchemaDependencyGraph + var maxLayer: Int + } + + struct ShardedOperationResult { + var files: [ShardedFile] + } + + private static let minDeclarationsPerFile = 12 + + private static func splitDeclarationsIntoFiles( + _ declarations: [Declaration], + maxFiles: Int + ) -> [[Declaration]] { + if maxFiles <= 1 { return [declarations] } + if declarations.isEmpty { return [[]] } + let minPerFile = (declarations.count + maxFiles - 1) / maxFiles + let perFile = max(minDeclarationsPerFile, minPerFile) + return stride(from: 0, to: declarations.count, by: perFile).map { start in + let end = min(start + perFile, declarations.count) + return Array(declarations[start.. String + ) -> [ShardedFile] { + let fileGroups = splitDeclarationsIntoFiles(declarations, maxFiles: maxFiles) + let totalFiles = maxFiles + return (0.. String { + switch self { + case .default: + return "Components_\(shard)_\(file).swift" + case .prefixed(let modulePrefix): + let compsBase = modulePrefix + "Components" + return "\(compsBase)_openapi_components_\(shard)_\(file).swift" + } + } + + func typeLayerShardFileName(layer: Int, shard: Int, file: Int) -> String { + switch self { + case .default: + return "Types_L\(layer + 1)_\(shard)_\(file).swift" + case .prefixed(let modulePrefix): + let suffix = typeLayerSuffix(layer: layer) + let layerBase = modulePrefix + suffix + return "\(layerBase)_openapi_\(suffix.lowercased())_\(shard)_\(file).swift" + } + } + + func operationLayerShardFileName(layer: Int, shard: Int, file: Int, isSingleFile: Bool = false) -> String { + switch self { + case .default: + if isSingleFile { return "Operations_L\(layer).swift" } + return "Operations_L\(layer)_\(shard)_\(file).swift" + case .prefixed(let modulePrefix): + let opsBase = (modulePrefix + "Operations").lowercased() + if isSingleFile { + return "\(opsBase)_openapi_operations_l\(layer).swift" + } + return "\(opsBase)_openapi_operations_l\(layer)_\(shard)_\(file).swift" + } + } + + var componentsBaseFileName: String { + switch self { + case .default: + return "Components_base.swift" + case .prefixed(let modulePrefix): + return "\(modulePrefix)Components_openapi_components.swift" + } + } + + var operationsBaseFileName: String { + switch self { + case .default: + return "Operations_base.swift" + case .prefixed(let modulePrefix): + return "\(modulePrefix)Operations_openapi_operations.swift" + } + } + } + + struct ShardImportResolver { + var config: ShardingConfig + var naming: ShardNamingStrategy + + private func componentBaseImports(prefix modulePrefix: String, exported: Bool = false) -> [ImportDescription] { + let compsBase = modulePrefix + "Components" + var result = [ImportDescription(moduleName: compsBase, exported: exported)] + for i in 1...config.typeShardCounts[0] { + result.append(ImportDescription(moduleName: "\(compsBase)_\(i)", exported: exported)) + } + return result + } + + private func typeLayerImports(prefix modulePrefix: String, upToLayer: Int, exported: Bool = false) -> [ImportDescription] { + var result: [ImportDescription] = [] + for layerIndex in 0.. [ImportDescription] { + guard case .prefixed(let modulePrefix) = naming else { return [] } + return [ImportDescription(moduleName: modulePrefix + "Components")] + } + + func typeLayerShardImports(layerIndex: Int) -> [ImportDescription] { + guard case .prefixed(let modulePrefix) = naming else { return [] } + return componentBaseImports(prefix: modulePrefix) + + typeLayerImports(prefix: modulePrefix, upToLayer: layerIndex) + } + + func exportedImportsForRootFile(maxLayer: Int) -> [ImportDescription] { + guard case .prefixed(let modulePrefix) = naming else { return [] } + let typeLayerCount = min(maxLayer, config.layerCount - 1) + var result = componentBaseImports(prefix: modulePrefix, exported: true) + + typeLayerImports(prefix: modulePrefix, upToLayer: typeLayerCount, exported: true) + let opsBase = modulePrefix + "Operations" + result.append(ImportDescription(moduleName: opsBase, exported: true)) + for layerIndex in 0...maxLayer { + let shardCount = config.operationLayerShardCounts[layerIndex] + if shardCount > 1 { + for s in 1...shardCount { + result.append(ImportDescription(moduleName: "\(opsBase)_L\(layerIndex)_\(s)", exported: true)) + } + } else { + result.append(ImportDescription(moduleName: "\(opsBase)_L\(layerIndex)", exported: true)) + } + } + return result + } + + func operationShardImports(layerIndex: Int) -> [ImportDescription] { + guard case .prefixed(let modulePrefix) = naming else { return [] } + var result = [ImportDescription(moduleName: "\(modulePrefix)Operations")] + result += componentBaseImports(prefix: modulePrefix) + result += typeLayerImports(prefix: modulePrefix, upToLayer: min(layerIndex, config.layerCount - 1)) + return result + } + } + + func translateSchemasSharded( + _ schemas: OpenAPI.ComponentDictionary, + multipartSchemaNames: Set, + shardingConfig: ShardingConfig, + naming: ShardNamingStrategy + ) throws -> ShardedSchemaResult { + let graph = SchemaDependencyGraph.build(from: schemas) + var declsBySchemaName: [String: [Declaration]] = [:] + for (key, value) in schemas { + let schemaName = key.rawValue + let decls = try translateSchema( + componentKey: key, + schema: value, + isMultipartContent: multipartSchemaNames.contains(key) + ) + declsBySchemaName[schemaName] = decls + } + + let maxLayer = min(shardingConfig.layerCount - 1, max(0, graph.layerCount - 1)) + + var islandsByLayer: [Int: [GraphAlgorithms.Island]] = [:] + for (compId, members) in graph.scc.components.enumerated() { + let layer = min(graph.layerOf[compId], maxLayer) + islandsByLayer[layer, default: []].append(members) + } + + let orderedSchemaDecls: [(name: String, decls: [Declaration])] = schemas.map { key, _ in + (name: key.rawValue, decls: declsBySchemaName[key.rawValue] ?? []) + } + let allDecls = orderedSchemaDecls.flatMap(\.decls) + let allBoxedDecls = try boxRecursiveTypes(allDecls) + guard allBoxedDecls.count == allDecls.count else { + throw GenericError( + message: "boxRecursiveTypes changed declaration count: \(allDecls.count) → \(allBoxedDecls.count)" + ) + } + var boxedDeclsBySchemaName: [String: [Declaration]] = [:] + var boxedIndex = 0 + for (name, decls) in orderedSchemaDecls { + boxedDeclsBySchemaName[name] = Array(allBoxedDecls[boxedIndex.. ShardedOperationResult { + var operationDeclsByLayer: [Int: [(operationID: String, declaration: Declaration)]] = [:] + + for description in operationDescriptions { + let schemaRefs = SchemaDependencyGraph.operationSchemaRefs( + description.operation, + in: description.components + ) + + let operationLayer = schemaRefs.compactMap { ref in + guard let layer = schemaGraph.layer(of: ref) else { return nil as Int? } + return min(layer, maxLayer) + }.max() ?? 0 + + let declaration = try translateOperation(description) + operationDeclsByLayer[operationLayer, default: []].append( + (operationID: description.operationID, declaration: declaration) + ) + } + + var files: [ShardedFile] = [] + + for layerIndex in 0...maxLayer { + let operations = operationDeclsByLayer[layerIndex] ?? [] + let shardCount = shardingConfig.operationLayerShardCounts[layerIndex] + let maxFiles = shardingConfig.maxFilesPerShardOps + + let islands: [GraphAlgorithms.Island] = operations.map { [$0.operationID] } + let declsByOpID = Dictionary(uniqueKeysWithValues: operations.map { ($0.operationID, $0.declaration) }) + + let bins = GraphAlgorithms.lptPacking(islands: islands, binCount: shardCount) { island in + let totalWeight = island.reduce(into: 0) { partialResult, opID in + if let decl = declsByOpID[opID] { + partialResult += declarationNodeCount(decl) + } else { + partialResult += 1 + } + } + return max(1, totalWeight) + } + + for (shardIndex, bin) in bins.enumerated() { + let opIDs = bin.flatMap { $0 }.sorted() + let shardDecls = opIDs.compactMap { declsByOpID[$0] } + + let isSingleFile = shardCount == 1 && maxFiles <= 1 + files += Self.emitPaddedFiles( + declarations: shardDecls, + maxFiles: maxFiles, + layer: layerIndex, + shardIndex: shardIndex + ) { fileIndex in + naming.operationLayerShardFileName( + layer: layerIndex, + shard: shardIndex + 1, + file: fileIndex + 1, + isSingleFile: isSingleFile + ) + } + } + } + + return ShardedOperationResult(files: files) + } } diff --git a/Sources/swift-openapi-generator/GenerateOptions+runGenerator.swift b/Sources/swift-openapi-generator/GenerateOptions+runGenerator.swift index 632dc1e6d..ef35d1047 100644 --- a/Sources/swift-openapi-generator/GenerateOptions+runGenerator.swift +++ b/Sources/swift-openapi-generator/GenerateOptions+runGenerator.swift @@ -37,6 +37,7 @@ extension _GenerateOptions { let resolvedNameOverrides = resolvedNameOverrides(config) let resolvedTypeOverrides = resolvedTypeOverrides(config) let resolvedFeatureFlags = resolvedFeatureFlags(config) + let resolvedSharding = try resolvedShardingConfig(config) let configs: [Config] = sortedModes.map { .init( mode: $0, @@ -47,7 +48,8 @@ extension _GenerateOptions { namingStrategy: resolvedNamingStragy, nameOverrides: resolvedNameOverrides, typeOverrides: resolvedTypeOverrides, - featureFlags: resolvedFeatureFlags + featureFlags: resolvedFeatureFlags, + sharding: $0 == .types ? resolvedSharding : nil ) } let (diagnostics, finalizeDiagnostics) = preparedDiagnosticsCollector(outputPath: diagnosticsOutputPath) diff --git a/Sources/swift-openapi-generator/GenerateOptions.swift b/Sources/swift-openapi-generator/GenerateOptions.swift index 182a2e921..98c2ef8d1 100644 --- a/Sources/swift-openapi-generator/GenerateOptions.swift +++ b/Sources/swift-openapi-generator/GenerateOptions.swift @@ -48,6 +48,10 @@ struct _GenerateOptions: ParsableArguments { @Option( help: "When specified, writes out the diagnostics into a YAML file instead of emitting them to standard error." ) var diagnosticsOutputPath: URL? + + @Option( + help: "Shard the Types output into multiple files by dependency layer. Format: comma-separated shards per layer (e.g. 4,4,2,2,1)" + ) var sharding: String? } extension AccessModifier: ExpressibleByArgument {} @@ -151,6 +155,26 @@ extension _GenerateOptions { return config?.featureFlags ?? [] } + func resolvedShardingConfig(_ config: _UserConfig?) throws -> ShardingConfig? { + if let sharding { + let shardCounts = try sharding.split(separator: ",").map { token -> Int in + guard let value = Int(token) else { + throw ValidationError("Invalid --sharding token '\(token)'. Expected comma-separated integers (e.g. 4,4,2,2,1)") + } + return value + } + guard !shardCounts.isEmpty else { + throw ValidationError("Invalid --sharding format. Expected comma-separated shards per layer (e.g. 4,4,2,2,1)") + } + let resolved = ShardingConfig(typeShardCounts: shardCounts, operationLayerShardCounts: shardCounts) + try resolved.validate() + return resolved + } + let resolved = config?.sharding + try resolved?.validate() + return resolved + } + /// Validates a collection of keys against a predefined set of allowed keys. /// /// - Parameter keys: A collection of keys to be validated. diff --git a/Sources/swift-openapi-generator/UserConfig.swift b/Sources/swift-openapi-generator/UserConfig.swift index f83dd71fd..38ab0422f 100644 --- a/Sources/swift-openapi-generator/UserConfig.swift +++ b/Sources/swift-openapi-generator/UserConfig.swift @@ -51,6 +51,9 @@ struct _UserConfig: Codable { /// A set of features to explicitly enable. var featureFlags: FeatureFlags? + /// Sharding configuration for splitting Types output into multiple files. + var sharding: ShardingConfig? + /// A set of raw values corresponding to the coding keys of this struct. static let codingKeysRawValues = Set(CodingKeys.allCases.map({ $0.rawValue })) @@ -64,6 +67,7 @@ struct _UserConfig: Codable { case nameOverrides case typeOverrides case featureFlags + case sharding } /// A container of type overrides. diff --git a/Sources/swift-openapi-generator/runGenerator.swift b/Sources/swift-openapi-generator/runGenerator.swift index 08b62d469..b3651f171 100644 --- a/Sources/swift-openapi-generator/runGenerator.swift +++ b/Sources/swift-openapi-generator/runGenerator.swift @@ -105,19 +105,35 @@ extension _Tool { isDryRun: Bool, diagnostics: any DiagnosticCollector ) throws { - try replaceFileContents( - inDirectory: outputDirectory, - fileName: outputFileName, - with: { - let output = try _OpenAPIGeneratorCore.runGenerator( - input: .init(absolutePath: doc, contents: docData), - config: config, - diagnostics: diagnostics + if config.sharding != nil { + let outputs = try _OpenAPIGeneratorCore.runShardedGenerator( + input: .init(absolutePath: doc, contents: docData), + config: config, + diagnostics: diagnostics + ) + for output in outputs { + try replaceFileContents( + inDirectory: outputDirectory, + fileName: output.baseName, + with: { output.contents }, + isDryRun: isDryRun ) - return output.contents - }, - isDryRun: isDryRun - ) + } + } else { + try replaceFileContents( + inDirectory: outputDirectory, + fileName: outputFileName, + with: { + let output = try _OpenAPIGeneratorCore.runGenerator( + input: .init(absolutePath: doc, contents: docData), + config: config, + diagnostics: diagnostics + ) + return output.contents + }, + isDryRun: isDryRun + ) + } } /// Evaluates a closure to generate file data and writes the data to disk diff --git a/Tests/OpenAPIGeneratorCoreTests/Test_GraphAlgorithms.swift b/Tests/OpenAPIGeneratorCoreTests/Test_GraphAlgorithms.swift new file mode 100644 index 000000000..3f3c3efda --- /dev/null +++ b/Tests/OpenAPIGeneratorCoreTests/Test_GraphAlgorithms.swift @@ -0,0 +1,271 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftOpenAPIGenerator open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftOpenAPIGenerator project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftOpenAPIGenerator project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +import XCTest +@testable import _OpenAPIGeneratorCore + +final class Test_GraphAlgorithms: XCTestCase { + + // MARK: - Tarjan SCC + + func testTarjanSCC_noCycles() { + // A -> B -> C (linear chain, no cycles) + let graph: [String: Set] = [ + "A": ["B"], + "B": ["C"], + "C": [], + ] + let result = GraphAlgorithms.tarjanSCC(graph: graph) + + XCTAssertEqual(result.components.count, 3) + for component in result.components { + XCTAssertEqual(component.count, 1) + } + } + + func testTarjanSCC_singleCycle() { + // A -> B -> C -> A (all in one SCC) + let graph: [String: Set] = [ + "A": ["B"], + "B": ["C"], + "C": ["A"], + ] + let result = GraphAlgorithms.tarjanSCC(graph: graph) + + XCTAssertEqual(result.components.count, 1) + XCTAssertEqual(result.components[0].sorted(), ["A", "B", "C"]) + } + + func testTarjanSCC_diamond_noCycles() { + // A -> B, A -> C, B -> D, C -> D + let graph: [String: Set] = [ + "A": ["B", "C"], + "B": ["D"], + "C": ["D"], + "D": [], + ] + let result = GraphAlgorithms.tarjanSCC(graph: graph) + + XCTAssertEqual(result.components.count, 4) + for component in result.components { + XCTAssertEqual(component.count, 1) + } + } + + func testTarjanSCC_mixed() { + // A -> B -> C -> B (cycle in B,C), A -> D (no cycle) + let graph: [String: Set] = [ + "A": ["B", "D"], + "B": ["C"], + "C": ["B"], + "D": [], + ] + let result = GraphAlgorithms.tarjanSCC(graph: graph) + + XCTAssertEqual(result.components.count, 3) + let cycleComponent = result.components.first { $0.count > 1 } + XCTAssertEqual(cycleComponent?.sorted(), ["B", "C"]) + } + + func testTarjanSCC_emptyGraph() { + let graph: [String: Set] = [:] + let result = GraphAlgorithms.tarjanSCC(graph: graph) + + XCTAssertEqual(result.components.count, 0) + } + + func testTarjanSCC_isolatedNodes() { + let graph: [String: Set] = [ + "A": [], + "B": [], + "C": [], + ] + let result = GraphAlgorithms.tarjanSCC(graph: graph) + + XCTAssertEqual(result.components.count, 3) + for component in result.components { + XCTAssertEqual(component.count, 1) + } + } + + // MARK: - Topological Sort + + func testTopologicalSort_linearChain() { + // 0 <- 1 <- 2 + let predecessors: [[Int]] = [[], [0], [1]] + let result = GraphAlgorithms.topologicalSort(predecessors: predecessors) + + XCTAssertEqual(result, [0, 1, 2]) + } + + func testTopologicalSort_diamond() { + // 0 -> 1, 0 -> 2, 1 -> 3, 2 -> 3 + let predecessors: [[Int]] = [[], [0], [0], [1, 2]] + let result = GraphAlgorithms.topologicalSort(predecessors: predecessors) + + XCTAssertEqual(result.first, 0) + XCTAssertEqual(result.last, 3) + XCTAssertTrue(result.firstIndex(of: 1)! < result.firstIndex(of: 3)!) + XCTAssertTrue(result.firstIndex(of: 2)! < result.firstIndex(of: 3)!) + } + + func testTopologicalSort_multipleRoots() { + // 0 and 1 are independent roots, both -> 2 + let predecessors: [[Int]] = [[], [], [0, 1]] + let result = GraphAlgorithms.topologicalSort(predecessors: predecessors) + + XCTAssertEqual(result.count, 3) + XCTAssertEqual(result.last, 2) + } + + func testTopologicalSort_empty() { + let result = GraphAlgorithms.topologicalSort(predecessors: []) + + XCTAssertEqual(result, []) + } + + // MARK: - Longest-Path Layering + + func testLongestPathLayering_linearChain() { + // 0 <- 1 <- 2 + let predecessors: [[Int]] = [[], [0], [1]] + let layers = GraphAlgorithms.longestPathLayering(dagPredecessors: predecessors) + + XCTAssertEqual(layers, [0, 1, 2]) + } + + func testLongestPathLayering_diamond() { + // 0 -> 1, 0 -> 2, 1 -> 3, 2 -> 3 + let predecessors: [[Int]] = [[], [0], [0], [1, 2]] + let layers = GraphAlgorithms.longestPathLayering(dagPredecessors: predecessors) + + XCTAssertEqual(layers[0], 0) + XCTAssertEqual(layers[1], 1) + XCTAssertEqual(layers[2], 1) + XCTAssertEqual(layers[3], 2) + } + + func testLongestPathLayering_multipleRoots() { + // 0 and 1 independent, 2 depends on both + let predecessors: [[Int]] = [[], [], [0, 1]] + let layers = GraphAlgorithms.longestPathLayering(dagPredecessors: predecessors) + + XCTAssertEqual(layers[0], 0) + XCTAssertEqual(layers[1], 0) + XCTAssertEqual(layers[2], 1) + } + + func testLongestPathLayering_singleNode() { + let predecessors: [[Int]] = [[]] + let layers = GraphAlgorithms.longestPathLayering(dagPredecessors: predecessors) + + XCTAssertEqual(layers, [0]) + } + + // MARK: - LPT Bin Packing + + func testLPTPacking_balanced() { + let islands: [GraphAlgorithms.Island] = [ + ["a", "b"], + ["c", "d"], + ["e", "f"], + ["g", "h"], + ] + let bins = GraphAlgorithms.lptPacking(islands: islands, binCount: 2) { $0.count } + + XCTAssertEqual(bins.count, 2) + let totalPerBin = bins.map { $0.flatMap { $0 }.count } + XCTAssertEqual(totalPerBin[0], 4) + XCTAssertEqual(totalPerBin[1], 4) + } + + func testLPTPacking_skewed() { + let islands: [GraphAlgorithms.Island] = [ + ["a", "b", "c", "d", "e"], + ["f"], + ["g"], + ] + let bins = GraphAlgorithms.lptPacking(islands: islands, binCount: 2) { $0.count } + + XCTAssertEqual(bins.count, 2) + let totalPerBin = bins.map { $0.flatMap { $0 }.count } + XCTAssertEqual(totalPerBin.sorted(), [2, 5]) + } + + func testLPTPacking_empty() { + let bins = GraphAlgorithms.lptPacking(islands: [], binCount: 3) { $0.count } + + XCTAssertEqual(bins.count, 3) + for bin in bins { + XCTAssertTrue(bin.isEmpty) + } + } + + func testLPTPacking_singleItem() { + let islands: [GraphAlgorithms.Island] = [ + ["a", "b", "c"], + ] + let bins = GraphAlgorithms.lptPacking(islands: islands, binCount: 3) { $0.count } + + XCTAssertEqual(bins.count, 3) + let nonEmpty = bins.filter { !$0.isEmpty } + XCTAssertEqual(nonEmpty.count, 1) + XCTAssertEqual(nonEmpty[0].flatMap { $0 }.count, 3) + } + + func testLPTPacking_zeroBins() { + let islands: [GraphAlgorithms.Island] = [["a"]] + let bins = GraphAlgorithms.lptPacking(islands: islands, binCount: 0) { $0.count } + + XCTAssertEqual(bins.count, 0) + } + + func testLPTPacking_customWeight() { + let islands: [GraphAlgorithms.Island] = [ + ["a"], + ["b"], + ["c"], + ] + let weights = ["a": 10, "b": 5, "c": 5] + let bins = GraphAlgorithms.lptPacking(islands: islands, binCount: 2) { island in + island.reduce(0) { $0 + (weights[$1] ?? 0) } + } + + XCTAssertEqual(bins.count, 2) + let binWeights = bins.map { bin in + bin.flatMap { $0 }.reduce(0) { $0 + (weights[$1] ?? 0) } + } + XCTAssertEqual(binWeights.sorted(), [10, 10]) + } + + // MARK: - Condensation DAG + + func testCondensationDAG_withCycle() { + // A -> B -> C -> B (cycle), A -> D + let graph: [String: Set] = [ + "A": ["B", "D"], + "B": ["C"], + "C": ["B"], + "D": [], + ] + let scc = GraphAlgorithms.tarjanSCC(graph: graph) + let dag = GraphAlgorithms.buildCondensationDAG(graph: graph, scc: scc) + + // Cycle {B,C} becomes single node, so 3 condensed nodes + XCTAssertEqual(dag.count, 3) + // Verify DAG has no self-loops + for (i, preds) in dag.enumerated() { + XCTAssertFalse(preds.contains(i)) + } + } +} diff --git a/Tests/OpenAPIGeneratorCoreTests/Test_ShardedGeneration.swift b/Tests/OpenAPIGeneratorCoreTests/Test_ShardedGeneration.swift new file mode 100644 index 000000000..c49a8ff7e --- /dev/null +++ b/Tests/OpenAPIGeneratorCoreTests/Test_ShardedGeneration.swift @@ -0,0 +1,192 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftOpenAPIGenerator open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftOpenAPIGenerator project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftOpenAPIGenerator project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +import XCTest +import Foundation +@testable import _OpenAPIGeneratorCore + +final class Test_ShardedGeneration: XCTestCase { + + /// A minimal OpenAPI spec with 5 schemas across 2+ dependency layers and 2 operations. + private static let specYAML = """ + openapi: "3.1.0" + info: + title: "Sharding Test API" + version: "1.0.0" + paths: + /items: + get: + operationId: listItems + responses: + "200": + description: OK + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Item" + /items/{id}: + get: + operationId: getItem + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/Item" + components: + schemas: + Color: + type: string + enum: [red, green, blue] + Tag: + type: object + properties: + name: + type: string + Category: + type: object + properties: + label: + type: string + color: + $ref: "#/components/schemas/Color" + Item: + type: object + properties: + name: + type: string + category: + $ref: "#/components/schemas/Category" + tags: + type: array + items: + $ref: "#/components/schemas/Tag" + DetailedItem: + type: object + properties: + item: + $ref: "#/components/schemas/Item" + description: + type: string + """ + + func testShardedGenerationProducesExpectedFiles() throws { + let config = Config( + mode: .types, + access: .public, + namingStrategy: .defensive, + sharding: ShardingConfig( + typeShardCounts: [1, 1, 1], + maxFilesPerShard: 1, + maxFilesPerShardOps: 1, + operationLayerShardCounts: [1, 1, 1] + ) + ) + + let input = InMemoryInputFile( + absolutePath: URL(string: "openapi.yaml")!, + contents: Data(Self.specYAML.utf8) + ) + let diagnostics = AccumulatingDiagnosticCollector() + let outputs = try runShardedGenerator( + input: input, + config: config, + diagnostics: diagnostics + ) + + let fileNames = outputs.map(\.baseName) + let fileNameSet = Set(fileNames) + + // Must have the root types file + XCTAssertTrue(fileNameSet.contains("Types_root.swift"), "Missing Types_root.swift, got: \(fileNames)") + + // Must have exactly one component base file + XCTAssertEqual( + fileNames.filter { $0 == "Components_base.swift" }.count, 1, + "Expected exactly one Components_base.swift, got: \(fileNames)" + ) + + // Must have component shard files (excluding the base file) + let componentShardFiles = fileNames.filter { + $0.hasPrefix("Components_") && $0 != "Components_base.swift" + } + XCTAssertEqual(componentShardFiles.count, 1, "Expected 1 component shard file (1 shard × 1 file), got: \(componentShardFiles)") + + // Must have operation files + let operationBaseFiles = fileNames.filter { $0 == "Operations_base.swift" } + XCTAssertEqual(operationBaseFiles.count, 1, "Expected exactly one Operations_base.swift, got: \(fileNames)") + + let operationLayerFiles = fileNames.filter { + $0.hasPrefix("Operations_L") || $0.hasPrefix("Operations_") && $0 != "Operations_base.swift" + } + XCTAssertEqual(operationLayerFiles.count, 3, "Expected 3 operation layer files (L0, L1, L2), got: \(operationLayerFiles)") + + // Verify Types_root.swift contains the API protocol + let rootFile = try XCTUnwrap(outputs.first { $0.baseName == "Types_root.swift" }) + let rootContent = String(data: rootFile.contents, encoding: .utf8)! + XCTAssertTrue(rootContent.contains("protocol APIProtocol")) + + // No diagnostics expected for a valid spec + XCTAssertEqual(diagnostics.diagnostics.count, 0) + } + + func testShardedGenerationWithPrefixedNaming() throws { + let config = Config( + mode: .types, + access: .public, + namingStrategy: .defensive, + sharding: ShardingConfig( + typeShardCounts: [1, 1, 1], + maxFilesPerShard: 1, + maxFilesPerShardOps: 1, + operationLayerShardCounts: [1, 1, 1], + modulePrefix: "TestModule" + ) + ) + + let input = InMemoryInputFile( + absolutePath: URL(string: "openapi.yaml")!, + contents: Data(Self.specYAML.utf8) + ) + let diagnostics = AccumulatingDiagnosticCollector() + let outputs = try runShardedGenerator( + input: input, + config: config, + diagnostics: diagnostics + ) + + let fileNames = outputs.map(\.baseName) + + // Prefixed naming should produce module-prefixed filenames + let componentFiles = fileNames.filter { $0.contains("TestModuleComponents") } + XCTAssertGreaterThanOrEqual(componentFiles.count, 1, "Expected TestModule-prefixed component files") + + // Should have operation files with module prefix + let operationFiles = fileNames.filter { $0.contains("testmoduleoperations") } + XCTAssertGreaterThanOrEqual(operationFiles.count, 1, "Expected TestModule-prefixed operation files") + + // Types_root should have @_exported imports + let rootFile = try XCTUnwrap(outputs.first { $0.baseName == "Types_root.swift" }) + let rootContent = String(data: rootFile.contents, encoding: .utf8)! + XCTAssertTrue(rootContent.contains("@_exported import"), "Types_root.swift should contain @_exported imports") + } +} diff --git a/Tests/OpenAPIGeneratorCoreTests/Test_Sharding.swift b/Tests/OpenAPIGeneratorCoreTests/Test_Sharding.swift new file mode 100644 index 000000000..9ffa62cd7 --- /dev/null +++ b/Tests/OpenAPIGeneratorCoreTests/Test_Sharding.swift @@ -0,0 +1,200 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftOpenAPIGenerator open source project +// +// Copyright (c) 2023 Apple Inc. and the SwiftOpenAPIGenerator project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftOpenAPIGenerator project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +import XCTest +import OpenAPIKit +@testable import _OpenAPIGeneratorCore + +final class Test_Sharding: Test_Core { + + // MARK: - Sharding Invariants + + private func makeShardingConfig( + typeShardCounts: [Int] = [2, 2, 1], + maxFilesPerShard: Int = 1, + maxFilesPerShardOps: Int = 1, + operationLayerShardCounts: [Int] = [1, 1, 1], + modulePrefix: String? = nil + ) -> ShardingConfig { + ShardingConfig( + typeShardCounts: typeShardCounts, + maxFilesPerShard: maxFilesPerShard, + maxFilesPerShardOps: maxFilesPerShardOps, + operationLayerShardCounts: operationLayerShardCounts, + modulePrefix: modulePrefix + ) + } + + private func makeSchemas() -> OpenAPI.ComponentDictionary { + // Create a small schema dependency tree: + // A (leaf), B (leaf), C -> A, D -> B, E -> C,D + // L0 (leaves): A, B + // L1: C, D (depend on L0) + // L2: E (depends on L1) + [ + "A": .object(properties: ["name": .string]), + "B": .object(properties: ["value": .integer]), + "C": .object(properties: [ + "a_ref": .reference(.component(named: "A")), + ]), + "D": .object(properties: [ + "b_ref": .reference(.component(named: "B")), + ]), + "E": .object(properties: [ + "c_ref": .reference(.component(named: "C")), + "d_ref": .reference(.component(named: "D")), + ]), + ] + } + + func testAllSchemasAssignedExactlyOnce() throws { + let schemas = makeSchemas() + let config = makeShardingConfig() + let translator = makeTranslator( + components: .init(schemas: schemas.mapValues { _ in .string }) + ) + + let result = try translator.translateSchemasSharded( + schemas, + multipartSchemaNames: [], + shardingConfig: config, + naming: .default + ) + + // Every shard file should have non-negative layer/shard/file indices + for file in result.files { + XCTAssertGreaterThanOrEqual(file.layer, 0) + XCTAssertGreaterThanOrEqual(file.shardIndex, 0) + XCTAssertGreaterThanOrEqual(file.fileIndex, 0) + XCTAssertFalse(file.fileName.isEmpty) + } + + // Files should span exactly 3 layers (L0, L1, L2) given our schema set + let layers = Set(result.files.map(\.layer)) + XCTAssertEqual(layers, [0, 1, 2]) + + // Every file with declarations should have non-empty content + let totalDeclCount = result.files.reduce(0) { $0 + $1.declarations.count } + XCTAssertGreaterThan(totalDeclCount, 0, "Expected declarations across shard files") + } + + func testDependencyGraphNoForwardReferences() { + let schemas = makeSchemas() + let graph = SchemaDependencyGraph.build(from: schemas) + + // Verify: for each schema, its dependencies are at a lower or equal layer + for (schemaName, deps) in graph.edges { + guard let schemaLayer = graph.layer(of: schemaName) else { continue } + for dep in deps { + guard let depLayer = graph.layer(of: dep) else { continue } + XCTAssertLessThanOrEqual( + depLayer, schemaLayer, + "Schema '\(schemaName)' (layer \(schemaLayer)) depends on '\(dep)' (layer \(depLayer))" + ) + } + } + } + + func testDeterministicOutput() throws { + let schemas = makeSchemas() + let config = makeShardingConfig() + + func runTranslation() throws -> [TypesFileTranslator.ShardedFile] { + let translator = makeTranslator( + components: .init(schemas: schemas.mapValues { _ in .string }) + ) + let result = try translator.translateSchemasSharded( + schemas, + multipartSchemaNames: [], + shardingConfig: config, + naming: .default + ) + return result.files + } + + let run1 = try runTranslation() + let run2 = try runTranslation() + + XCTAssertEqual(run1.count, run2.count) + for (f1, f2) in zip(run1, run2) { + XCTAssertEqual(f1.fileName, f2.fileName) + XCTAssertEqual(f1.layer, f2.layer) + XCTAssertEqual(f1.shardIndex, f2.shardIndex) + XCTAssertEqual(f1.fileIndex, f2.fileIndex) + XCTAssertEqual(f1.declarations, f2.declarations) + } + } + + // MARK: - File Naming Contract + + func testPrefixedNamingProducesExpectedPatterns() { + let modulePrefix = "MyServiceAPI" + let naming = TypesFileTranslator.ShardNamingStrategy.prefixed( + modulePrefix: modulePrefix + ) + + // --- Components base --- + XCTAssertEqual( + naming.componentsBaseFileName, + "MyServiceAPIComponents_openapi_components.swift" + ) + + // --- Component shard files --- + XCTAssertEqual( + naming.componentShardFileName(shard: 1, file: 1), + "MyServiceAPIComponents_openapi_components_1_1.swift" + ) + XCTAssertEqual( + naming.componentShardFileName(shard: 3, file: 2), + "MyServiceAPIComponents_openapi_components_3_2.swift" + ) + + // --- Type layer shard files --- + for layerIndex in 0..<5 { + let suffix = "Types_L\(layerIndex + 1)" + let expected = "\(modulePrefix)\(suffix)_openapi_\(suffix.lowercased())_1_1.swift" + XCTAssertEqual( + naming.typeLayerShardFileName(layer: layerIndex, shard: 1, file: 1), + expected + ) + } + + // --- Operations base --- + XCTAssertEqual( + naming.operationsBaseFileName, + "MyServiceAPIOperations_openapi_operations.swift" + ) + + // --- Operation layer shard files (single shard) --- + XCTAssertEqual( + naming.operationLayerShardFileName(layer: 0, shard: 1, file: 1, isSingleFile: true), + "myserviceapioperations_openapi_operations_l0.swift" + ) + + // --- Operation layer shard files (multi shard) --- + XCTAssertEqual( + naming.operationLayerShardFileName(layer: 2, shard: 1, file: 1), + "myserviceapioperations_openapi_operations_l2_1_1.swift" + ) + } + + func testDefaultNamingProducesExpectedPatterns() { + let naming = TypesFileTranslator.ShardNamingStrategy.default + + XCTAssertEqual(naming.componentsBaseFileName, "Components_base.swift") + XCTAssertEqual(naming.operationsBaseFileName, "Operations_base.swift") + XCTAssertEqual(naming.componentShardFileName(shard: 2, file: 3), "Components_2_3.swift") + XCTAssertEqual(naming.typeLayerShardFileName(layer: 1, shard: 2, file: 1), "Types_L2_2_1.swift") + XCTAssertEqual(naming.operationLayerShardFileName(layer: 0, shard: 1, file: 2), "Operations_L0_1_2.swift") + } +} diff --git a/Tests/OpenAPIGeneratorReferenceTests/SnippetBasedReferenceTests.swift b/Tests/OpenAPIGeneratorReferenceTests/SnippetBasedReferenceTests.swift index d1e6341c0..9ac936fa8 100644 --- a/Tests/OpenAPIGeneratorReferenceTests/SnippetBasedReferenceTests.swift +++ b/Tests/OpenAPIGeneratorReferenceTests/SnippetBasedReferenceTests.swift @@ -6228,7 +6228,7 @@ final class SnippetBasedReferenceTests: XCTestCase { let document = try YAMLDecoder().decode(OpenAPI.Document.self, from: documentYAML) let translation = try translator.translateFile(parsedOpenAPI: document) try XCTAssertSwiftEquivalent( - XCTUnwrap(translation.file.contents.topComment), + XCTUnwrap(translation.files[0].contents.topComment), """ // Generated by swift-openapi-generator, do not modify. // hello world