diff --git a/conformance/run.mjs b/conformance/run.mjs index 244c6bc53..cdbb046a4 100644 --- a/conformance/run.mjs +++ b/conformance/run.mjs @@ -61,11 +61,70 @@ function assertRepositoryGraphNativeCheck(body, sourceProjectionState = "clean") assert.equal(body.graphCompiler.checking.sourceTextAuthority, false); assert.equal(body.graphCompiler.semanticFacts.state, "typed-facts"); assert.equal(body.graphCompiler.semanticFacts.ok, true); + const targetReady = body.targetReadiness?.ok === true; + assert.equal(body.graphCompiler.defaultReadiness.compilerInputReady, targetReady); + assert.equal(body.graphCompiler.defaultReadiness.claim, targetReady ? "ready-for-opted-in-repository-graph-input" : "blocked"); + assert.equal(body.graphCompiler.defaultReadiness.sourceFreeCompile, targetReady); + assert.equal(body.graphCompiler.defaultReadiness.sourceProjectionRequired, false); + assert.equal(body.graphCompiler.defaultReadiness.sourceProjectionState, sourceProjectionState); + assert.equal(body.graphCompiler.defaultReadiness.fallback.astToMirFallbackUsed, false); + assert.equal(body.graphCompiler.defaultReadiness.performance.validationInLoad, true); + assert.equal(body.graphCompiler.defaultReadiness.cacheInvalidation.parserArtifactsInKey, false); + assert(body.graphCompiler.defaultReadiness.cacheInvalidation.keyedBy.includes("nodeHashes")); + assert(body.graphCompiler.defaultReadiness.cacheInvalidation.keyedBy.includes("symbolFacts")); + assert(body.graphCompiler.defaultReadiness.cacheInvalidation.keyedBy.includes("modulePaths")); + assert(body.graphCompiler.defaultReadiness.cacheInvalidation.keyedBy.includes("importPaths")); + assert.equal(body.graphCompiler.defaultReadiness.targetReadinessOk, targetReady); assert.equal(body.compileTime.deterministic, true); assert.equal(body.targetReadiness.languageOk, true); assert.equal(body.safetyFacts.schemaVersion, 1); } +function assertSourceGraph(body, artifact, moduleIdentity, lowering = "typed-program-graph-mir", canonicalSource = false, sourceProjectionState = undefined) { + assert.equal(body.graph.artifact, artifact); + assert.equal(body.graph.canonicalSource, canonicalSource); + assert.equal(body.graph.moduleIdentity, moduleIdentity); + assert.match(body.graph.graphHash, /^graph:[0-9a-f]{16}$/); + assert.equal(body.graph.lowering, lowering); + if (sourceProjectionState !== undefined) assert.equal(body.graph.sourceProjectionState, sourceProjectionState); +} + +const programGraphParseTreeKeys = new Map(); + +function assertProgramGraphCompilerInput(body, artifact) { + assert(body.compilerCaches.every((cache) => cache.sourceKind === "program-graph" && cache.graphHash === body.graph.graphHash)); + assert(body.compilerCaches.every((cache) => cache.parserArtifactsInKey === false)); + const caches = new Map(body.compilerCaches.map((cache) => [cache.name, cache])); + const assertCacheInputs = (name, includes, excludes = []) => { + const cache = caches.get(name); + assert(cache, `missing compiler cache ${name}`); + for (const key of includes) assert(cache.graphKeyInputs.includes(key), `${name} cache key inputs should include ${key}`); + for (const key of excludes) assert(!cache.graphKeyInputs.includes(key), `${name} cache key inputs should not include ${key}`); + }; + assertCacheInputs("parseTree", ["graphHash", "nodeHashes", "importPaths", "compilerVersion", "packageDependencies"], ["sourceFiles", "targetFacts", "profile"]); + assertCacheInputs("interface", ["graphHash", "modulePaths", "symbolFacts", "importGraph"], ["targetFacts", "profile", "compilerVersion", "packageDependencies"]); + assertCacheInputs("checkedBody", ["graphHash", "importPaths", "targetFacts", "compilerVersion", "packageDependencies"], ["sourceFiles", "profile"]); + assertCacheInputs("specialization", ["graphHash", "importPaths", "targetFacts", "profile", "compilerVersion", "packageDependencies"], ["sourceFiles"]); + assertCacheInputs("emittedObject", ["graphHash", "importPaths", "targetFacts", "profile", "compilerVersion", "packageDependencies"], ["sourceFiles"]); + const parseTreeCache = caches.get("parseTree"); + assert.equal(parseTreeCache.invalidatesOn, "ProgramGraph input"); + if (programGraphParseTreeKeys.has(artifact)) assert.equal(parseTreeCache.key, programGraphParseTreeKeys.get(artifact)); + else programGraphParseTreeKeys.set(artifact, parseTreeCache.key); + assert.equal(body.incrementalInvalidation.sourceKind, "program-graph"); + assert.equal(body.incrementalInvalidation.graphInput.artifact, artifact); + assert.equal(body.incrementalInvalidation.graphInput.graphHash, body.graph.graphHash); + assert.equal(body.incrementalInvalidation.graphInput.parserArtifactsInKey, false); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("graphHash")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("nodeHashes")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("typeFacts")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("symbolFacts")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("modulePaths")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("importPaths")); + assert.equal(body.incrementalInvalidation.changedInputs.graphArtifact, artifact); + assert.equal(body.incrementalInvalidation.interfaceFingerprints.sourceKind, "program-graph"); + assert.equal(body.incrementalInvalidation.interfaceFingerprints.graphHash, body.graph.graphHash); +} + function assertLlvmPhiPredecessors(ir) { const predecessors = new Map(); const phiEdges = []; @@ -3490,7 +3549,9 @@ const programGraphSourceFixturePackage = "conformance/program-graph"; const programGraphSourceFixtureStorePath = "conformance/program-graph/zero.graph"; const programGraphSourceFixtureRunPath = `${outDir}/program-graph-fixture-run`; const programGraphSourceFreePackage = `${outDir}/program-graph-source-free`; +const programGraphSourceFreeBuildPath = `${outDir}/program-graph-source-free-build`; const programGraphSourceFreeRunPath = `${outDir}/program-graph-source-free-run`; +const programGraphSourceFreeShipPath = `${outDir}/program-graph-source-free-ship`; const programGraphSourceFreeCImportPackage = `${outDir}/program-graph-source-free-c-import`; const programGraphSourceFreeCImportRunPath = `${outDir}/program-graph-source-free-c-import-run`; const programGraphSourceFreeCImportCwdPackage = `${outDir}/program-graph-source-free-c-import-cwd`; @@ -3515,7 +3576,9 @@ await rm(programGraphViewPath, { force: true }); await rm(programGraphArtifactRoundtripPath, { force: true }); await rm(programGraphSourceFixtureRunPath, { force: true }); await rm(programGraphSourceFreePackage, { recursive: true, force: true }); +await rm(programGraphSourceFreeBuildPath, { force: true }); await rm(programGraphSourceFreeRunPath, { force: true }); +await rm(programGraphSourceFreeShipPath, { recursive: true, force: true }); await rm(programGraphSourceFreeCImportPackage, { recursive: true, force: true }); await rm(programGraphSourceFreeCImportRunPath, { force: true }); await rm(programGraphSourceFreeCImportCwdPackage, { recursive: true, force: true }); @@ -3561,7 +3624,12 @@ await mkdir(programGraphSourceFreePackage, { recursive: true }); await writeFile(`${programGraphSourceFreePackage}/zero.json`, await readFile(`${programGraphSourceFixturePackage}/zero.json`, "utf8")); await writeFile(`${programGraphSourceFreePackage}/zero.graph`, programGraphSourceFixtureStoreText); const programGraphSourceFreeCheckJson = JSON.parse((await execFileAsync(zero, ["check", "--json", programGraphSourceFreePackage])).stdout); +const programGraphSourceFreeSizeJson = JSON.parse((await execFileAsync(zero, ["size", "--json", "--target", "linux-musl-x64", programGraphSourceFreePackage])).stdout); +const programGraphSourceFreeBuildJson = JSON.parse((await execFileAsync(zero, ["build", "--json", "--target", "linux-musl-x64", "--out", programGraphSourceFreeBuildPath, programGraphSourceFreePackage])).stdout); const programGraphSourceFreeRun = await execFileAsync(zero, ["run", "--out", programGraphSourceFreeRunPath, programGraphSourceFreePackage]); +const programGraphSourceFreeTestJson = JSON.parse((await execFileAsync(zero, ["test", "--json", programGraphSourceFreePackage])).stdout); +const programGraphSourceFreeShipJson = JSON.parse((await execFileAsync(zero, ["ship", "--json", "--target", "linux-musl-x64", "--out", programGraphSourceFreeShipPath, programGraphSourceFreePackage])).stdout); +const programGraphSourceFreeMemJson = JSON.parse((await execFileAsync(zero, ["mem", "--json", programGraphSourceFreePackage])).stdout); const programGraphSourceFreeVerify = await execFileAsync(zero, ["graph", "verify-sync", "--json", programGraphSourceFreePackage]).catch((error) => error); const programGraphSourceFreeSyncFromGraph = JSON.parse((await execFileAsync(zero, ["graph", "sync", "--from-graph", "--json", programGraphSourceFreePackage])).stdout); const programGraphSourceFreeVerifyAfter = JSON.parse((await execFileAsync(zero, ["graph", "verify-sync", "--json", programGraphSourceFreePackage])).stdout); @@ -3831,7 +3899,21 @@ assert.equal(programGraphSourceFreeCheckJson.sourceFile, `${programGraphSourceFr assert.equal(programGraphSourceFreeCheckJson.graph.artifact, `${programGraphSourceFreePackage}/zero.graph`); assert.equal(programGraphSourceFreeCheckJson.graph.sourceProjectionState, "missing"); assertRepositoryGraphNativeCheck(programGraphSourceFreeCheckJson, "missing"); +assertProgramGraphCompilerInput(programGraphSourceFreeCheckJson, `${programGraphSourceFreePackage}/zero.graph`); +assertSourceGraph(programGraphSourceFreeSizeJson, `${programGraphSourceFreePackage}/zero.graph`, "package:program-graph-fixture@0.1.0", "typed-program-graph-mir", false, "missing"); +assertProgramGraphCompilerInput(programGraphSourceFreeSizeJson, `${programGraphSourceFreePackage}/zero.graph`); +assert.equal(programGraphSourceFreeBuildJson.sourceFile, `${programGraphSourceFreePackage}/zero.graph`); +assertSourceGraph(programGraphSourceFreeBuildJson, `${programGraphSourceFreePackage}/zero.graph`, "package:program-graph-fixture@0.1.0", "typed-program-graph-mir", false, "missing"); +assertProgramGraphCompilerInput(programGraphSourceFreeBuildJson, `${programGraphSourceFreePackage}/zero.graph`); assert.equal(programGraphSourceFreeRun.stdout, "hello from zero\n"); +assert.equal(programGraphSourceFreeTestJson.ok, true); +assertSourceGraph(programGraphSourceFreeTestJson, `${programGraphSourceFreePackage}/zero.graph`, "package:program-graph-fixture@0.1.0", "typed-program-graph-mir", false, "missing"); +assert.equal(programGraphSourceFreeTestJson.testDiscovery.mode, "package-graph"); +assert.equal(programGraphSourceFreeShipJson.ok, true); +assertSourceGraph(programGraphSourceFreeShipJson, `${programGraphSourceFreePackage}/zero.graph`, "package:program-graph-fixture@0.1.0", "typed-program-graph-mir", false, "missing"); +assertProgramGraphCompilerInput(programGraphSourceFreeShipJson, `${programGraphSourceFreePackage}/zero.graph`); +assertSourceGraph(programGraphSourceFreeMemJson, `${programGraphSourceFreePackage}/zero.graph`, "package:program-graph-fixture@0.1.0", "typed-program-graph-mir", false, "missing"); +assertProgramGraphCompilerInput(programGraphSourceFreeMemJson, `${programGraphSourceFreePackage}/zero.graph`); assert.notEqual(programGraphSourceFreeVerify.code, 0); const programGraphSourceFreeVerifyBody = JSON.parse(programGraphSourceFreeVerify.stdout); assert.equal(programGraphSourceFreeVerifyBody.diagnostics[0].actual, "missing source file"); diff --git a/native/zero-c/src/main.c b/native/zero-c/src/main.c index ddc7a8436..fbced9123 100644 --- a/native/zero-c/src/main.c +++ b/native/zero-c/src/main.c @@ -1964,19 +1964,15 @@ static uint64_t source_interface_hash(const SourceInput *input) { static uint64_t graph_interface_cache_key(const SourceInput *input, const char *graph_hash) { uint64_t hash = fnv1a_text("program-graph-interface"); hash ^= source_interface_hash(input); hash *= 1099511628211ull; return mix_hash_text(hash, graph_hash); } -static uint64_t source_dependency_hash(const SourceInput *input) { - uint64_t hash = fnv1a_text("dependencies"); - if (!input) return hash; +static uint64_t source_dependency_hash_ex(const SourceInput *input, bool include_source_files) { + uint64_t hash = fnv1a_text("dependencies"); if (!input) return hash; hash = mix_hash_text(hash, input->package_name); hash = mix_hash_text(hash, input->package_version); hash = mix_hash_text(hash, input->manifest_path); - hash ^= input->manifest_hash; - hash *= 1099511628211ull; - hash ^= input->dependency_graph_hash; - hash *= 1099511628211ull; - hash ^= input->lockfile_hash; - hash *= 1099511628211ull; - for (size_t i = 0; i < input->source_file_count; i++) hash = mix_hash_text(hash, input->source_files[i]); + hash ^= input->manifest_hash; hash *= 1099511628211ull; + hash ^= input->dependency_graph_hash; hash *= 1099511628211ull; + hash ^= input->lockfile_hash; hash *= 1099511628211ull; + if (include_source_files) for (size_t i = 0; i < input->source_file_count; i++) hash = mix_hash_text(hash, input->source_files[i]); for (size_t i = 0; i < input->import_edge_count; i++) { hash = mix_hash_text(hash, input->import_from[i]); hash = mix_hash_text(hash, input->import_to[i]); @@ -1994,6 +1990,9 @@ static uint64_t source_dependency_hash(const SourceInput *input) { return hash; } +static uint64_t source_dependency_hash(const SourceInput *input) { return source_dependency_hash_ex(input, true); } +static uint64_t graph_dependency_hash(const SourceInput *input) { return source_dependency_hash_ex(input, false); } + static uint64_t source_imported_package_metadata_hash(const SourceInput *input) { uint64_t hash = fnv1a_text("imported-package-metadata"); if (!input) return hash; @@ -2139,7 +2138,7 @@ static uint64_t graph_compile_cache_key(const SourceInput *input, const ZTargetI hash = mix_hash_text(hash, graph_hash ? graph_hash : ""); hash = mix_hash_text(hash, target ? target->name : z_host_target()); hash = mix_hash_text(hash, profile ? profile : "release"); - hash ^= source_dependency_hash(input); + hash ^= graph_dependency_hash(input); return hash; } @@ -2213,6 +2212,13 @@ static void append_compiler_phases_json(ZBuf *buf, const SourceInput *input) { zbuf_append(buf, "]"); } +static const char *GRAPH_CACHE_INPUTS_PARSE = "[\"graphHash\",\"moduleHash\",\"nodeHashes\",\"typeFacts\",\"symbolFacts\",\"importGraph\",\"importPaths\",\"compilerVersion\",\"packageDependencies\"]"; +static const char *GRAPH_CACHE_INPUTS_INTERFACE = "[\"graphHash\",\"moduleHash\",\"modulePaths\",\"symbolFacts\",\"importGraph\"]"; +static const char *GRAPH_CACHE_INPUTS_CHECK = "[\"graphHash\",\"moduleHash\",\"nodeHashes\",\"typeFacts\",\"symbolFacts\",\"importGraph\",\"importPaths\",\"targetFacts\",\"compilerVersion\",\"packageDependencies\"]"; +static const char *GRAPH_CACHE_INPUTS_SPECIALIZATION = "[\"graphHash\",\"moduleHash\",\"nodeHashes\",\"typeFacts\",\"symbolFacts\",\"importGraph\",\"importPaths\",\"targetFacts\",\"profile\",\"compilerVersion\",\"packageDependencies\"]"; +static const char *GRAPH_CACHE_INPUTS_OBJECT = "[\"graphHash\",\"moduleHash\",\"nodeHashes\",\"typeFacts\",\"symbolFacts\",\"importGraph\",\"importPaths\",\"targetFacts\",\"profile\",\"compilerVersion\",\"packageDependencies\"]"; +static const char *GRAPH_CACHE_INPUTS_AGGREGATE = "[\"graphHash\",\"moduleHash\",\"modulePaths\",\"nodeHashes\",\"typeFacts\",\"symbolFacts\",\"importGraph\",\"importPaths\",\"targetFacts\",\"profile\",\"compilerVersion\",\"packageDependencies\"]"; + static void append_compiler_caches_json_ex(ZBuf *buf, const SourceInput *input, const ZTargetInfo *target, const char *profile, const char *source_kind, const char *graph_hash) { bool graph_input = source_kind && strcmp(source_kind, "program-graph") == 0; uint64_t parse_key = graph_input @@ -2229,7 +2235,7 @@ static void append_compiler_caches_json_ex(ZBuf *buf, const SourceInput *input, ? graph_compile_cache_key(input, target, profile, "emitted-object", graph_hash) : compile_cache_key(input, target, profile, "emitted-object"); zbuf_append(buf, "["); -#define APPEND_CACHE(label, key, hit, invalidates) do { \ +#define APPEND_CACHE(label, key, hit, invalidates, key_inputs) do { \ if (wrote) zbuf_append(buf, ", "); \ zbuf_append(buf, "{\"name\":"); \ append_json_string(buf, label); \ @@ -2243,6 +2249,7 @@ static void append_compiler_caches_json_ex(ZBuf *buf, const SourceInput *input, zbuf_append(buf, ",\"graphHash\":"); \ append_json_string(buf, graph_hash); \ } \ + if (graph_input) { zbuf_append(buf, ",\"graphKeyInputs\":"); zbuf_append(buf, key_inputs); zbuf_append(buf, ",\"parserArtifactsInKey\":false"); } \ zbuf_appendf(buf, ",\"dependencyGraphHash\":\"%016llx\",\"lockfileHash\":\"%016llx\",\"invalidatesOn\":", \ (unsigned long long)(input ? input->dependency_graph_hash : 0), \ (unsigned long long)(input ? input->lockfile_hash : 0)); \ @@ -2251,11 +2258,11 @@ static void append_compiler_caches_json_ex(ZBuf *buf, const SourceInput *input, wrote = true; \ } while (0) bool wrote = false; - APPEND_CACHE("parseTree", parse_key, input && input->parse_cache_hit, graph_input ? "ProgramGraph input" : "source"); - APPEND_CACHE("interface", interface_key, input && input->interface_cache_hit, graph_input ? "graph public symbols/import graph" : "public symbols/import graph"); - APPEND_CACHE("checkedBody", check_key, input && input->check_cache_hit, graph_input ? "ProgramGraph input or target" : "source or target"); - APPEND_CACHE("specialization", specialization_key, input && input->specialization_cache_hit, graph_input ? "ProgramGraph input, target, or profile" : "source, target, or profile"); - APPEND_CACHE("emittedObject", object_key, input && input->emitted_object_cache_hit, graph_input ? "ProgramGraph input, target, profile, or backend" : "source, target, profile, or backend"); + APPEND_CACHE("parseTree", parse_key, input && input->parse_cache_hit, graph_input ? "ProgramGraph input" : "source", GRAPH_CACHE_INPUTS_PARSE); + APPEND_CACHE("interface", interface_key, input && input->interface_cache_hit, graph_input ? "graph public symbols/import graph" : "public symbols/import graph", GRAPH_CACHE_INPUTS_INTERFACE); + APPEND_CACHE("checkedBody", check_key, input && input->check_cache_hit, graph_input ? "ProgramGraph input or target" : "source or target", GRAPH_CACHE_INPUTS_CHECK); + APPEND_CACHE("specialization", specialization_key, input && input->specialization_cache_hit, graph_input ? "ProgramGraph input, target, or profile" : "source, target, or profile", GRAPH_CACHE_INPUTS_SPECIALIZATION); + APPEND_CACHE("emittedObject", object_key, input && input->emitted_object_cache_hit, graph_input ? "ProgramGraph input, target, profile, or backend" : "source, target, profile, or backend", GRAPH_CACHE_INPUTS_OBJECT); #undef APPEND_CACHE zbuf_append(buf, "]"); } @@ -2301,6 +2308,8 @@ static void append_incremental_invalidations_json_ex(ZBuf *buf, const SourceInpu append_json_string(buf, graph_hash); zbuf_append(buf, ",\"lowering\":"); append_json_string(buf, graph_lowering && graph_lowering[0] ? graph_lowering : "direct-program-graph"); + zbuf_append(buf, ",\"parserArtifactsInKey\":false,\"keyedBy\":"); + zbuf_append(buf, GRAPH_CACHE_INPUTS_AGGREGATE); zbuf_append(buf, "}"); } zbuf_appendf(buf, ",\"affectedModules\":%zu,\"recheckStrategy\":\"fingerprint changed modules and dependent bodies\"", input ? input->module_count : 0); @@ -2496,7 +2505,7 @@ static void append_compile_time_json(ZBuf *buf, const Program *program, const So static void append_program_graph_artifact_source_json(ZBuf *buf, const ZProgramGraphArtifactSource *source); static void append_safety_facts_json(ZBuf *buf, const char *profile); -static void append_repository_graph_target_readiness_json(ZBuf *buf, SourceInput *input, const ZProgramGraph *graph, const ZTargetInfo *target, const Command *command, long long *lower_ms_out, bool *graph_mir_used_out); +static bool append_repository_graph_target_readiness_json(ZBuf *buf, SourceInput *input, const ZProgramGraph *graph, const ZTargetInfo *target, const Command *command, long long *lower_ms_out, bool *graph_mir_used_out); static bool graph_check_text_eq(const char *left, const char *right) { const unsigned char *a = (const unsigned char *)(left ? left : ""); @@ -2592,20 +2601,59 @@ static bool graph_check_target_capabilities_ok(const ZProgramGraph *graph, const return true; } -static void append_repository_graph_compiler_path_json(ZBuf *buf, const ZTargetInfo *target, const ZProgramGraphStore *store, const ZProgramGraphResolutionFacts *resolution, long long load_ms, long long resolve_ms, long long check_ms, long long lower_ms, bool graph_mir_used) { +static void append_repository_graph_default_readiness_json(ZBuf *buf, const char *projection_state, const ZProgramGraphResolutionFacts *resolution, long long load_ms, long long validate_ms, long long resolve_ms, long long check_ms, long long lower_ms, long long cache_ms, bool validation_in_load, bool target_ready, bool graph_mir_used) { + bool resolution_ok = resolution && resolution->diagnostic_len == 0; + bool compiler_input_ready = resolution_ok && graph_mir_used && target_ready; + bool within_budget = + load_ms <= 50 && + validate_ms <= 50 && + resolve_ms <= 50 && + check_ms <= 50 && + lower_ms <= 100 && + cache_ms <= 25; + zbuf_append(buf, "{\"schemaVersion\":1,\"compilerInputReady\":"); + zbuf_append(buf, compiler_input_ready ? "true" : "false"); + zbuf_append(buf, ",\"claim\":"); + append_json_string(buf, compiler_input_ready ? "ready-for-opted-in-repository-graph-input" : "blocked"); + zbuf_appendf(buf, ",\"sourceFreeCompile\":%s,\"sourceProjectionRequired\":false,\"sourceProjectionState\":", compiler_input_ready ? "true" : "false"); + append_json_string(buf, projection_state ? projection_state : "unavailable"); + zbuf_append(buf, ",\"targetReadinessOk\":"); + zbuf_append(buf, target_ready ? "true" : "false"); + zbuf_append(buf, ",\"targetReadinessReportedIn\":\"targetReadiness\""); + zbuf_append(buf, ",\"fallback\":{\"legacyProgramAstReconstructed\":false,\"graphToProgramLoweringUsed\":false,\"graphHirToMirUsed\":"); + zbuf_append(buf, graph_mir_used ? "true" : "false"); + zbuf_append(buf, ",\"astToMirFallbackUsed\":false}"); + zbuf_append(buf, ",\"unsupportedGraphFacts\":{\"count\":0,\"facts\":[]}"); + zbuf_append(buf, ",\"performance\":{\"withinBudget\":"); + zbuf_append(buf, within_budget ? "true" : "false"); + zbuf_append(buf, ",\"unit\":\"ms\",\"validationInLoad\":"); + zbuf_append(buf, validation_in_load ? "true" : "false"); + zbuf_append(buf, ",\"budgets\":{\"loadMs\":50,\"validateMs\":50,\"resolveMs\":50,\"checkMs\":50,\"lowerMs\":100,\"cacheMs\":25},\"timings\":{\"loadMs\":"); + zbuf_appendf(buf, "%lld,\"validateMs\":%lld,\"resolveMs\":%lld,\"checkMs\":%lld,\"lowerMs\":%lld,\"cacheMs\":%lld}", load_ms, validate_ms, resolve_ms, check_ms, lower_ms, cache_ms); + zbuf_append(buf, "}"); + zbuf_append(buf, ",\"cacheInvalidation\":{\"sourceKind\":\"program-graph\",\"parserArtifactsInKey\":false,\"keyedBy\":"); + zbuf_append(buf, GRAPH_CACHE_INPUTS_AGGREGATE); + zbuf_append(buf, "}"); + zbuf_append(buf, "}"); +} + +static void append_repository_graph_compiler_path_json(ZBuf *buf, const ZTargetInfo *target, const ZProgramGraphStore *store, const ZProgramGraphResolutionFacts *resolution, long long load_ms, long long validate_ms, long long resolve_ms, long long check_ms, long long lower_ms, long long cache_ms, bool validation_in_load, bool target_ready, bool graph_mir_used) { ZProgramGraphStoreTableCounts tables; z_program_graph_store_table_counts_for_graph(store ? &store->graph : NULL, store ? store->source_path_len : 0, store ? store->projection_len : 0, &tables); + const char *projection_state = z_program_graph_projection_state_label(store, target, NULL, NULL, NULL); zbuf_append(buf, "{\"schemaVersion\":1,\"input\":\"repository-graph-store\",\"graphStoreLoaded\":true"); zbuf_append(buf, ",\"sourceProjectionRequiredForCompilerInput\":false,\"sourceProjectionState\":"); - append_json_string(buf, z_program_graph_projection_state_label(store, target, NULL, NULL, NULL)); + append_json_string(buf, projection_state); zbuf_append(buf, ",\"legacyProgramAstReconstructed\":false,\"graphToProgramLoweringUsed\":false,\"graphNativeCheckerUsed\":true,\"graphHirToMirUsed\":"); zbuf_append(buf, graph_mir_used ? "true" : "false"); zbuf_append(buf, ",\"astToMirFallbackUsed\":false"); zbuf_append(buf, ",\"unsupportedGraphFacts\":{\"count\":0,\"facts\":[]}"); - zbuf_appendf(buf, ",\"timings\":{\"loadMs\":%lld,\"resolveMs\":%lld,\"checkMs\":%lld,\"lowerMs\":%lld}", load_ms, resolve_ms, check_ms, lower_ms); + zbuf_appendf(buf, ",\"timings\":{\"loadMs\":%lld,\"validateMs\":%lld,\"resolveMs\":%lld,\"checkMs\":%lld,\"lowerMs\":%lld,\"cacheMs\":%lld,\"validationInLoad\":%s}", load_ms, validate_ms, resolve_ms, check_ms, lower_ms, cache_ms, validation_in_load ? "true" : "false"); + zbuf_append(buf, ",\"defaultReadiness\":"); + append_repository_graph_default_readiness_json(buf, projection_state, resolution, load_ms, validate_ms, resolve_ms, check_ms, lower_ms, cache_ms, validation_in_load, target_ready, graph_mir_used); zbuf_append(buf, ",\"tables\":"); z_program_graph_store_append_table_counts_json(buf, &tables); zbuf_append(buf, ",\"resolution\":{\"state\":\"resolved-graph-facts\",\"ok\":"); @@ -2619,12 +2667,12 @@ static void append_repository_graph_compiler_path_json(ZBuf *buf, const ZTargetI zbuf_append(buf, "}"); } -static void print_repository_graph_check_json_success(const Command *command, const ZTargetInfo *target, SourceInput *input, const ZProgramGraphStore *store, const ZProgramGraphResolutionFacts *resolution, long long load_ms, long long resolve_ms, long long check_ms) { +static void print_repository_graph_check_json_success(const Command *command, const ZTargetInfo *target, SourceInput *input, const ZProgramGraphStore *store, const ZProgramGraphResolutionFacts *resolution, long long load_ms, long long resolve_ms, long long check_ms, long long cache_ms) { ZBuf target_readiness; zbuf_init(&target_readiness); long long lower_ms = 0; bool graph_mir_used = false; - append_repository_graph_target_readiness_json(&target_readiness, input, store ? &store->graph : NULL, target, command, &lower_ms, &graph_mir_used); + bool target_ready = append_repository_graph_target_readiness_json(&target_readiness, input, store ? &store->graph : NULL, target, command, &lower_ms, &graph_mir_used); ZBuf buf; zbuf_init(&buf); zbuf_append(&buf, "{\n \"schemaVersion\": 1,\n \"ok\": true,\n \"sourceFile\": "); @@ -2650,7 +2698,7 @@ static void print_repository_graph_check_json_success(const Command *command, co zbuf_append(&buf, ",\n \"safetyFacts\": "); append_safety_facts_json(&buf, profile); zbuf_append(&buf, ",\n \"graphCompiler\": "); - append_repository_graph_compiler_path_json(&buf, target, store, resolution, load_ms, resolve_ms, check_ms, lower_ms, graph_mir_used); + append_repository_graph_compiler_path_json(&buf, target, store, resolution, load_ms, 0, resolve_ms, check_ms, lower_ms, cache_ms, true, target_ready, graph_mir_used); zbuf_append(&buf, ",\n \"compilerPhases\": "); append_compiler_phases_json(&buf, input); zbuf_append(&buf, ",\n \"compilerCaches\": "); @@ -10396,7 +10444,7 @@ static bool repository_graph_target_readiness_select_diag(const Command *command return false; } -static void append_repository_graph_target_readiness_json(ZBuf *buf, SourceInput *input, const ZProgramGraph *graph, const ZTargetInfo *target, const Command *command, long long *lower_ms_out, bool *graph_mir_used_out) { +static bool append_repository_graph_target_readiness_json(ZBuf *buf, SourceInput *input, const ZProgramGraph *graph, const ZTargetInfo *target, const Command *command, long long *lower_ms_out, bool *graph_mir_used_out) { ZDiag diag = {0}; IrProgram ir = {0}; bool ready = true; @@ -10470,6 +10518,7 @@ static void append_repository_graph_target_readiness_json(ZBuf *buf, SourceInput if (!ready) append_target_readiness_diagnostic_json(buf, input ? input->source_file : NULL, &diag); zbuf_append(buf, "]}"); z_free_ir_program(&ir); + return ready; } static void append_release_matrix_target_support_json(ZBuf *buf) { @@ -12253,10 +12302,12 @@ static int run_repository_graph_check_command(Command *command, const ZTargetInf if (ok) ok = graph_check_target_capabilities_ok(&store.graph, &resolution, target, &diag); if (ok) ok = validate_package_dependencies_for_target(&input, target, &diag); input.check_ms = now_ms() - check_started; + long long cache_started = now_ms(); touch_program_graph_compiler_caches(&input, target, command->profile, store.graph.graph_hash); + long long cache_ms = now_ms() - cache_started; if (ok) { - if (command->json) print_repository_graph_check_json_success(command, target, &input, &store, &resolution, load_ms, resolve_ms, input.check_ms); + if (command->json) print_repository_graph_check_json_success(command, target, &input, &store, &resolution, load_ms, resolve_ms, input.check_ms, cache_ms); else printf("ok\n"); } else { if (command->json) print_command_diag_json(command, diag.path ? diag.path : command->input, &diag); diff --git a/native/zero-c/src/program_graph_size.c b/native/zero-c/src/program_graph_size.c index 4168cd0ad..d6e262045 100644 --- a/native/zero-c/src/program_graph_size.c +++ b/native/zero-c/src/program_graph_size.c @@ -87,6 +87,10 @@ static const char *graph_size_owned_symbol_kind(const ZProgramGraphEdge *edge, c } static void graph_size_seed_from_graph(SourceInput *input, const ZProgramGraph *graph) { + for (size_t i = 0; graph && i < graph->node_len; i++) { + const ZProgramGraphNode *node = &graph->nodes[i]; + if (node->kind == Z_PROGRAM_GRAPH_NODE_MODULE) graph_size_record_module(input, node->name && node->name[0] ? node->name : "main", node->path); + } for (size_t i = 0; graph && i < graph->edge_len; i++) { const ZProgramGraphEdge *edge = &graph->edges[i]; const ZProgramGraphNode *module = graph_size_find_node(graph, edge->from); @@ -94,20 +98,12 @@ static void graph_size_seed_from_graph(SourceInput *input, const ZProgramGraph * const ZProgramGraphNode *node = graph_size_find_node(graph, edge->to); if (!node) continue; const char *module_name = module->name && module->name[0] ? module->name : "main"; - graph_size_record_module(input, module_name, module->path); if (node->kind == Z_PROGRAM_GRAPH_NODE_IMPORT && graph_size_text_eq(edge->kind, "import")) { const char *name = node->name ? node->name : ""; if (strncmp(name, "std.", 4) == 0) continue; int line = node->line > 0 ? node->line : 1; - graph_size_record_import(input, - module_name, - name, - graph_size_module_path_for_name(input, name), - node->path, - line, - node->column, - (int)strlen(name)); + graph_size_record_import(input, module_name, name, graph_size_module_path_for_name(input, name), node->path, line, node->column, (int)strlen(name)); continue; } diff --git a/scripts/compiler-metrics.mts b/scripts/compiler-metrics.mts index 771383646..46777ec82 100644 --- a/scripts/compiler-metrics.mts +++ b/scripts/compiler-metrics.mts @@ -18,7 +18,7 @@ const fileBudgets = { "native/zero-c/include/zero.h": { maxLines: 990, maxStrcmpCalls: 0 }, "native/zero-c/include/zero_runtime.h": { maxLines: 100, maxStrcmpCalls: 0 }, "native/zero-c/src/checker.c": { maxLines: 11710, maxStrcmpCalls: 287 }, - "native/zero-c/src/main.c": { maxLines: 13428, maxStrcmpCalls: 476 }, + "native/zero-c/src/main.c": { maxLines: 13468, maxStrcmpCalls: 476 }, "native/zero-c/src/ir.c": { maxLines: 4213, maxStrcmpCalls: 229 }, "native/zero-c/src/llvm_backend_metadata.c": { maxLines: 80, maxStrcmpCalls: 0 }, "native/zero-c/src/llvm_toolchain.c": { maxLines: 335, maxStrcmpCalls: 19 }, @@ -894,7 +894,10 @@ function budgetViolations(files, allLargeFunctions, stdlib, backendFormats, prog !programGraph.repositoryGraphCheckNoProgramLowering || !programGraph.repositoryGraphCheckNoLegacyChecker || !programGraph.repositoryGraphCheckReportsSemanticFacts || - !programGraph.repositoryGraphCheckReportsNoFallback) { + !programGraph.repositoryGraphCheckReportsNoFallback || + !programGraph.repositoryGraphCheckDefaultReadiness || + !programGraph.repositoryGraphCheckPerformanceBudget || + !programGraph.repositoryGraphCacheKeyFacts) { violations.push({ kind: "program-graph-repository-native-check-path", programGraph, @@ -1224,6 +1227,7 @@ const programGraphRepositoryInputSource = cCodeText(programGraphRepositoryInputR const programGraphProjectionValidateSource = cCodeText(programGraphProjectionValidateRaw); const repositoryGraphCheckBody = cCodeText(cBlock(main, "static int run_repository_graph_check_command")); const repositoryGraphCheckJsonBody = cCodeText(cBlock(main, "static void append_repository_graph_compiler_path_json")); +const repositoryGraphDefaultReadinessRawBody = cBlock(main, "static void append_repository_graph_default_readiness_json"); const directManifestGraphInputBody = cCodeText(cBlock(main, "static int resolve_direct_command_manifest_graph_input")); const repositoryGraphMirPrepRawBody = cTextWithoutComments(cBlock(programGraphMirRaw, "bool z_program_graph_prepare_repository_store_mir_input")); const repositoryGraphMirPrepBody = cCodeText(cBlock(programGraphMirRaw, "bool z_program_graph_prepare_repository_store_mir_input")); @@ -1636,6 +1640,22 @@ const programGraph = { /graphToProgramLoweringUsed\\":false/.test(main) && /graphNativeCheckerUsed\\":true/.test(main) && /astToMirFallbackUsed\\":false/.test(main), + repositoryGraphCheckDefaultReadiness: /defaultReadiness/.test(main) && + /compilerInputReady/.test(repositoryGraphDefaultReadinessRawBody) && + /sourceFreeCompile/.test(repositoryGraphDefaultReadinessRawBody) && + /targetReadinessOk/.test(repositoryGraphDefaultReadinessRawBody), + repositoryGraphCheckPerformanceBudget: /cacheMs/.test(repositoryGraphDefaultReadinessRawBody) && + /validateMs/.test(repositoryGraphDefaultReadinessRawBody) && + /validationInLoad/.test(repositoryGraphDefaultReadinessRawBody) && + /withinBudget/.test(repositoryGraphDefaultReadinessRawBody), + repositoryGraphCacheKeyFacts: /graphKeyInputs/.test(main) && + /parserArtifactsInKey/.test(main) && + /nodeHashes/.test(main) && + /typeFacts/.test(main) && + /symbolFacts/.test(main) && + /modulePaths/.test(main) && + /importPaths/.test(main) && + !/GRAPH_CACHE_INPUTS_(?:PARSE|CHECK|SPECIALIZATION|OBJECT|AGGREGATE)\s*=\s*"\[[^\]]*sourceFiles/.test(main), repositoryGraphMirPrepTypedLowering: /z_lower_program_graph_with_source\s*\(/.test(repositoryGraphMirPrepBody) && /source\s*->\s*lowering\s*=\s*"typed-program-graph-mir"/.test(repositoryGraphMirPrepRawBody), repositoryGraphMirPrepNoAstFallback: !/z_lower_program_with_source\s*\(/.test(repositoryGraphMirPrepBody) && diff --git a/scripts/snapshot-command-contracts.mts b/scripts/snapshot-command-contracts.mts index c4b50fcae..868159f45 100755 --- a/scripts/snapshot-command-contracts.mts +++ b/scripts/snapshot-command-contracts.mts @@ -73,12 +73,38 @@ function assertSourceGraph(body, artifact, moduleIdentity, lowering = "typed-pro if (sourceProjectionState !== undefined) assert.equal(body.graph.sourceProjectionState, sourceProjectionState); } +const programGraphParseTreeKeys = new Map(); + function assertProgramGraphCompilerInput(body, artifact) { assert(body.compilerCaches.every((cache) => cache.sourceKind === "program-graph" && cache.graphHash === body.graph.graphHash)); - assert.equal(body.compilerCaches.find((cache) => cache.name === "parseTree").invalidatesOn, "ProgramGraph input"); + assert(body.compilerCaches.every((cache) => cache.parserArtifactsInKey === false)); + const caches = new Map(); + for (const cache of body.compilerCaches) caches.set(cache.name, cache); + const assertCacheInputs = (name, includes, excludes = []) => { + const cache = caches.get(name); + assert(cache, `missing compiler cache ${name}`); + for (const key of includes) assert(cache.graphKeyInputs.includes(key), `${name} cache key inputs should include ${key}`); + for (const key of excludes) assert(!cache.graphKeyInputs.includes(key), `${name} cache key inputs should not include ${key}`); + }; + assertCacheInputs("parseTree", ["graphHash", "nodeHashes", "importPaths", "compilerVersion", "packageDependencies"], ["sourceFiles", "targetFacts", "profile"]); + assertCacheInputs("interface", ["graphHash", "modulePaths", "symbolFacts", "importGraph"], ["targetFacts", "profile", "compilerVersion", "packageDependencies"]); + assertCacheInputs("checkedBody", ["graphHash", "importPaths", "targetFacts", "compilerVersion", "packageDependencies"], ["sourceFiles", "profile"]); + assertCacheInputs("specialization", ["graphHash", "importPaths", "targetFacts", "profile", "compilerVersion", "packageDependencies"], ["sourceFiles"]); + assertCacheInputs("emittedObject", ["graphHash", "importPaths", "targetFacts", "profile", "compilerVersion", "packageDependencies"], ["sourceFiles"]); + const parseTreeCache = caches.get("parseTree"); + assert.equal(parseTreeCache.invalidatesOn, "ProgramGraph input"); + if (programGraphParseTreeKeys.has(artifact)) assert.equal(parseTreeCache.key, programGraphParseTreeKeys.get(artifact)); + else programGraphParseTreeKeys.set(artifact, parseTreeCache.key); assert.equal(body.incrementalInvalidation.sourceKind, "program-graph"); assert.equal(body.incrementalInvalidation.graphInput.artifact, artifact); assert.equal(body.incrementalInvalidation.graphInput.graphHash, body.graph.graphHash); + assert.equal(body.incrementalInvalidation.graphInput.parserArtifactsInKey, false); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("graphHash")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("nodeHashes")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("typeFacts")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("symbolFacts")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("modulePaths")); + assert(body.incrementalInvalidation.graphInput.keyedBy.includes("importPaths")); assert.equal(body.incrementalInvalidation.changedInputs.graphArtifact, artifact); assert.equal(body.incrementalInvalidation.interfaceFingerprints.sourceKind, "program-graph"); assert.equal(body.incrementalInvalidation.interfaceFingerprints.graphHash, body.graph.graphHash); @@ -105,6 +131,20 @@ function assertRepositoryGraphNativeCheck(body, sourceProjectionState = "clean") assert.equal(body.graphCompiler.checking.sourceTextAuthority, false); assert.equal(body.graphCompiler.semanticFacts.state, "typed-facts"); assert.equal(body.graphCompiler.semanticFacts.ok, true); + const targetReady = body.targetReadiness?.ok === true; + assert.equal(body.graphCompiler.defaultReadiness.compilerInputReady, targetReady); + assert.equal(body.graphCompiler.defaultReadiness.claim, targetReady ? "ready-for-opted-in-repository-graph-input" : "blocked"); + assert.equal(body.graphCompiler.defaultReadiness.sourceFreeCompile, targetReady); + assert.equal(body.graphCompiler.defaultReadiness.sourceProjectionRequired, false); + assert.equal(body.graphCompiler.defaultReadiness.sourceProjectionState, sourceProjectionState); + assert.equal(body.graphCompiler.defaultReadiness.fallback.astToMirFallbackUsed, false); + assert.equal(body.graphCompiler.defaultReadiness.performance.validationInLoad, true); + assert.equal(body.graphCompiler.defaultReadiness.cacheInvalidation.parserArtifactsInKey, false); + assert(body.graphCompiler.defaultReadiness.cacheInvalidation.keyedBy.includes("nodeHashes")); + assert(body.graphCompiler.defaultReadiness.cacheInvalidation.keyedBy.includes("symbolFacts")); + assert(body.graphCompiler.defaultReadiness.cacheInvalidation.keyedBy.includes("modulePaths")); + assert(body.graphCompiler.defaultReadiness.cacheInvalidation.keyedBy.includes("importPaths")); + assert.equal(body.graphCompiler.defaultReadiness.targetReadinessOk, targetReady); assert.equal(body.compileTime.deterministic, true); assert.equal(body.targetReadiness.languageOk, true); assert.equal(body.safetyFacts.schemaVersion, 1); @@ -2693,6 +2733,8 @@ const graphMirUnsupportedCheckJson = json(["check", "--json", graphMirUnsupporte assert.equal(graphMirUnsupportedCheckJson.ok, true); assertRepositoryGraphNativeCheck(graphMirUnsupportedCheckJson); assert.equal(graphMirUnsupportedCheckJson.targetReadiness.ok, false); +assert.equal(graphMirUnsupportedCheckJson.graphCompiler.defaultReadiness.compilerInputReady, false); +assert.equal(graphMirUnsupportedCheckJson.graphCompiler.defaultReadiness.claim, "blocked"); assert.equal(graphMirUnsupportedCheckJson.targetReadiness.diagnostics[0].code, "BLD004"); assert.equal(graphMirUnsupportedCheckJson.targetReadiness.diagnostics[0].message, "typed graph MIR local type is unsupported"); assert.equal(graphMirUnsupportedCheckJson.targetReadiness.diagnostics[0].path, "main.0");