Skip to content

Commit 08056a9

Browse files
cknittclaude
andcommitted
Recompile dependents of a changed CMJ in the OCaml build system
Why: The OCaml build system, now the default `rescript`, recompiled dependents only when a module's CMI changed. An `@inline(crossModule)` body lives in the CMJ and can change while the CMI stays the same, so callers kept the old inlined code, during a build and on the next build after the dependency was compiled separately. Rust rewatch already compares CMJ digests since the previous commits. Publishing a byte-different CMJ now marks dependents dirty like a changed CMI, and the startup freshness check compares a dependent's CMT with the later of its dependency's CMI and CMJ. Review: Does every implementation change now recompile dependents? Only those whose CMJ bytes change, matching the Rust implementation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Kbf3gKsWykY6KH4QSJ9Wrs Signed-off-by: Christoph Knittel <ck@cca.io>
1 parent 7ce5284 commit 08056a9

7 files changed

Lines changed: 56 additions & 14 deletions

File tree

‎rewatch-ocaml/build_state.ml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,7 @@ let record_published_cmi ?dirty_propagation state ~compile_assets module_ ~path
111111
mark_dependents_compile_dirty ?visited:dirty_propagation state module_);
112112
Compile_assets.refresh_cmi compile_assets ~key:module_.key ~path;
113113
module_.last_compiled_cmi <-
114-
Compile_assets.cmi compile_assets module_.key
115-
|> Option.map (fun entry -> entry.Compile_assets.modified)
114+
Compile_assets.published compile_assets module_.key
116115

117116
let record_successful_compile ~compile_assets module_ ~cmt_path =
118117
Compile_assets.refresh_cmt compile_assets ~key:module_.key ~path:cmt_path;

‎rewatch-ocaml/build_state.mli‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ type module_ = {
1515
mutable dependents: String_set.t;
1616
mutable compile_dirty: bool;
1717
mutable last_compiled_cmi: float option;
18+
(** See {!Compile_assets.published}. *)
1819
mutable last_compiled_cmt: float option;
1920
}
2021

‎rewatch-ocaml/compile_assets.ml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ type t = {
88
ast_by_source: (string, entry) Hashtbl.t;
99
cmi_by_module: (string, entry) Hashtbl.t;
1010
cmt_by_module: (string, entry) Hashtbl.t;
11+
cmj_by_module: (string, entry) Hashtbl.t;
1112
}
1213

1314
let ast_header path =
@@ -21,7 +22,7 @@ let is_managed_basename basename =
2122
List.exists (Filename.check_suffix basename) cleanup_extensions
2223

2324
let state_extension = function
24-
| ".ast" | ".iast" | ".cmi" | ".cmt" -> true
25+
| ".ast" | ".iast" | ".cmi" | ".cmj" | ".cmt" -> true
2526
| _ -> false
2627

2728
let read_directory directory =
@@ -58,6 +59,7 @@ let add_module_artifact state (entry, name) =
5859
match Filename.extension name with
5960
| ".cmi" -> Hashtbl.replace state.cmi_by_module (module_key name) entry
6061
| ".cmt" -> Hashtbl.replace state.cmt_by_module (module_key name) entry
62+
| ".cmj" -> Hashtbl.replace state.cmj_by_module (module_key name) entry
6163
| _ -> ()
6264

6365
let create directories =
@@ -69,6 +71,7 @@ let create directories =
6971
ast_by_source = Hashtbl.create 64;
7072
cmi_by_module = Hashtbl.create 64;
7173
cmt_by_module = Hashtbl.create 64;
74+
cmj_by_module = Hashtbl.create 64;
7275
}
7376
in
7477
directories
@@ -118,14 +121,22 @@ let ast state source = Hashtbl.find_opt state.ast_by_source (source_key source)
118121
let cmi state key = Hashtbl.find_opt state.cmi_by_module key
119122
let cmt state key = Hashtbl.find_opt state.cmt_by_module key
120123

124+
let published state key =
125+
match (cmi state key, Hashtbl.find_opt state.cmj_by_module key) with
126+
| None, _ -> None
127+
| Some cmi, None -> Some cmi.modified
128+
| Some cmi, Some cmj -> Some (Float.max cmi.modified cmj.modified)
129+
121130
let replace_from_path table key path =
122131
try
123132
Hashtbl.replace table key {path; modified = (Unix.stat path).Unix.st_mtime}
124133
with Unix.Unix_error ((Unix.ENOENT | Unix.ENOTDIR), _, _) ->
125134
Hashtbl.remove table key
126135

127136
let refresh_cmi state ~key ~path =
128-
replace_from_path state.cmi_by_module key path
137+
replace_from_path state.cmi_by_module key path;
138+
replace_from_path state.cmj_by_module key
139+
(Filename.remove_extension path ^ ".cmj")
129140

130141
let refresh_cmt state ~key ~path =
131142
replace_from_path state.cmt_by_module key path

‎rewatch-ocaml/compile_assets.mli‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,14 @@ val ast_dependencies : t -> string -> string list
1414
val ast : t -> string -> entry option
1515
val cmi : t -> string -> entry option
1616
val cmt : t -> string -> entry option
17+
18+
val published : t -> string -> float option
19+
(** When the module's CMI, or its CMJ if newer, was last written; [None] without
20+
a CMI. A CMJ can change without its CMI, for example when an exported inline
21+
body changes, and dependents compiled before it are stale too. *)
22+
1723
val refresh_cmi : t -> key:string -> path:string -> unit
24+
(** Also refreshes the CMJ next to [path]. *)
25+
1826
val refresh_cmt : t -> key:string -> path:string -> unit
1927
val refresh_ast : t -> source:string -> path:string -> unit

‎rewatch-ocaml/compiler_process.ml‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,11 @@ let parse_request ~build_dir ~(config : Config.t) path =
124124
let ast_dependencies ~build_dir ast =
125125
(Ast_header.read (Filename.concat build_dir ast)).dependencies
126126

127-
type compiler_artifact = Cmi | Required of string | Optional of string
127+
type compiler_artifact = Cmi | Cmj | Required of string | Optional of string
128+
129+
(* The returned change tells whether dependents must recompile. Besides the
130+
.cmi, this includes the .cmj: exported inline bodies can change without
131+
changing the module's type. *)
128132

129133
let publish_compiler_artifacts ~artifact_dir ~ocaml_dir ~basename artifacts =
130134
let cmi_change = ref Compiler_scheduler.Cmi_change_unknown in
@@ -134,6 +138,7 @@ let publish_compiler_artifacts ~artifact_dir ~ocaml_dir ~basename artifacts =
134138
let extension =
135139
match artifact with
136140
| Cmi -> "cmi"
141+
| Cmj -> "cmj"
137142
| Required name | Optional name -> name
138143
in
139144
let source =
@@ -148,6 +153,14 @@ let publish_compiler_artifacts ~artifact_dir ~ocaml_dir ~basename artifacts =
148153
if File_util.copy_file_if_different source destination then
149154
Compiler_scheduler.Cmi_changed
150155
else Compiler_scheduler.Cmi_unchanged
156+
| Cmj -> (
157+
match
158+
(File_util.copy_file_if_different source destination, !cmi_change)
159+
with
160+
| true, _ -> cmi_change := Compiler_scheduler.Cmi_changed
161+
| false, Compiler_scheduler.Cmi_change_unknown ->
162+
cmi_change := Compiler_scheduler.Cmi_unchanged
163+
| false, (Compiler_scheduler.Cmi_changed | Cmi_unchanged) -> ())
151164
| Required _ -> File_util.copy_existing_file source destination
152165
| Optional _ -> File_util.copy_optional_existing_file source destination)
153166
artifacts;
@@ -278,13 +291,11 @@ let publish ~build_dir ~ocaml_dir ~is_local ~(config : Config.t) ~has_interface
278291
| Source.Implementation when has_interface ->
279292
(* The implementation is compiled with -bs-read-cmi, so its .cmi is
280293
the one already published after compiling the interface. *)
281-
ignore
282-
(publish_compiler_artifacts ~artifact_dir ~ocaml_dir ~basename
283-
[Required "cmj"; Optional "cmt"]);
284-
Compiler_scheduler.Cmi_unchanged
294+
publish_compiler_artifacts ~artifact_dir ~ocaml_dir ~basename
295+
[Cmj; Optional "cmt"]
285296
| Source.Implementation ->
286297
publish_compiler_artifacts ~artifact_dir ~ocaml_dir ~basename
287-
[Cmi; Required "cmj"; Optional "cmt"]);
298+
[Cmi; Cmj; Optional "cmt"]);
288299
let source = Filename.concat config.root path in
289300
let build_source = Filename.concat build_dir path in
290301
File_util.ensure_dir (Filename.dirname build_source);

‎rewatch-ocaml/module_graph.ml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -291,8 +291,7 @@ let initialize ~(root_config : Config.t) ~package_plans ~compile_assets
291291
List.iter
292292
(fun ((node : module_node), _) ->
293293
Build_state.add build_state ~key:node.key ~kind:Build_state.Source_module
294-
~last_compiled_cmi:
295-
(Compile_assets.cmi compile_assets node.key |> modified)
294+
~last_compiled_cmi:(Compile_assets.published compile_assets node.key)
296295
~last_compiled_cmt:
297296
(Compile_assets.cmt compile_assets node.key |> modified))
298297
source_graph_nodes;
@@ -301,8 +300,7 @@ let initialize ~(root_config : Config.t) ~package_plans ~compile_assets
301300
Build_state.add build_state ~key:namespace_map.key
302301
~kind:Build_state.Namespace_map
303302
~last_compiled_cmi:
304-
(Compile_assets.cmi compile_assets namespace_map.compiler_name
305-
|> modified)
303+
(Compile_assets.published compile_assets namespace_map.compiler_name)
306304
~last_compiled_cmt:
307305
(Compile_assets.cmt compile_assets namespace_map.compiler_name
308306
|> modified))

‎tests/rewatch_ounit_tests/compile_assets_tests.ml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,25 @@ let tests =
109109
check
110110
(Option.is_none (Compile_assets.cmi state "Nested"))
111111
"the compiler asset directory is scanned non-recursively";
112+
Unix.utimes first 100. 100.;
113+
Unix.utimes cleanup_only 200. 200.;
114+
Compile_assets.refresh_cmi state ~key:"Example" ~path:first;
115+
check
116+
(Compile_assets.published state "Example" = Some 200.)
117+
"a CMJ newer than its CMI counts as a publication";
118+
Unix.utimes first 300. 300.;
119+
Compile_assets.refresh_cmi state ~key:"Example" ~path:first;
120+
check
121+
(Compile_assets.published state "Example" = Some 300.)
122+
"an older CMJ leaves the CMI publication time";
112123
Sys.remove first;
113124
Compile_assets.refresh_cmi state ~key:"Example" ~path:first;
114125
check
115126
(Option.is_none (Compile_assets.cmi state "Example"))
116127
"refresh removes a deleted CMI";
128+
check
129+
(Option.is_none (Compile_assets.published state "Example"))
130+
"a CMJ without a CMI is not a complete publication";
117131
let replacement_ast = Filename.concat root "Replacement.ast" in
118132
write replacement_ast "replacement";
119133
Compile_assets.refresh_ast state ~source ~path:replacement_ast;

0 commit comments

Comments
 (0)