-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathcompatibility.ex
More file actions
178 lines (145 loc) · 6.32 KB
/
Copy pathcompatibility.ex
File metadata and controls
178 lines (145 loc) · 6.32 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
defmodule Snakepit.Compatibility do
@moduledoc """
Thread-safety compatibility matrix for common Python libraries.
> #### Legacy Optional Module {: .warning}
>
> `Snakepit` does not call this module internally. It remains available for
> compatibility and may be removed in `v0.16.0` or later.
>
> Prefer explicit worker-profile and adapter-level thread-safety
> configuration in your host application.
"""
alias Snakepit.Internal.Deprecation
@type thread_safety :: true | false | :conditional
@type library_info :: %{
thread_safe: thread_safety(),
notes: String.t()
}
@legacy_replacement "Use worker profile and adapter-level thread-safety settings"
@legacy_remove_after "v0.16.0"
@libraries %{
"numpy" => %{thread_safe: true, notes: "Releases GIL during computation"},
"scipy" => %{thread_safe: true, notes: "Releases GIL for numerical ops"},
"torch" => %{thread_safe: true, notes: "Configure with torch.set_num_threads/1"},
"tensorflow" => %{thread_safe: true, notes: "Use tf.config.threading APIs"},
"scikit-learn" => %{thread_safe: :conditional, notes: "Set n_jobs=1 per estimator"},
"polars" => %{thread_safe: true, notes: "Thread-safe DataFrame library"},
"requests" => %{thread_safe: true, notes: "Use separate Session per thread"},
"httpx" => %{thread_safe: true, notes: "Async-first; thread-safe clients"},
"aiohttp" => %{thread_safe: :conditional, notes: "One ClientSession per thread"},
"grpcio" => %{thread_safe: true, notes: "Thread-safe client with shared channels"},
"numexpr" => %{thread_safe: true, notes: "Releases GIL for expression evaluation"},
"onnxruntime" => %{thread_safe: true, notes: "Thread-safe inference sessions"},
"jax" => %{thread_safe: :conditional, notes: "Avoid shared mutable state"},
"opencv" => %{thread_safe: :conditional, notes: "Avoid shared mutable state"},
"pillow" => %{thread_safe: :conditional, notes: "Use separate Image objects"},
"pandas" => %{thread_safe: false, notes: "Not thread-safe; lock DataFrame ops"},
"matplotlib" => %{thread_safe: false, notes: "Global state; prefer process mode"},
"sqlite3" => %{thread_safe: false, notes: "Use separate connections per thread"},
"sqlalchemy" => %{thread_safe: :conditional, notes: "Use per-thread sessions"},
"spacy" => %{thread_safe: false, notes: "Models share global state"},
"fasttext" => %{thread_safe: false, notes: "Model objects are not thread-safe"},
"xgboost" => %{thread_safe: :conditional, notes: "Set num_threads and avoid shared state"},
"lightgbm" => %{thread_safe: :conditional, notes: "Set num_threads and avoid shared state"},
"cupy" => %{thread_safe: :conditional, notes: "Manage CUDA context per thread"},
"faiss" => %{thread_safe: :conditional, notes: "Avoid shared index mutation"},
"ray" => %{thread_safe: false, notes: "Prefer process-based workers"},
"celery" => %{thread_safe: false, notes: "Use process-based workers"}
}
@spec check(String.t() | atom(), :thread | :process) ::
{:ok, String.t()} | {:warning, String.t()} | {:error, String.t()}
def check(library, profile) when profile in [:thread, :process] do
mark_legacy_usage()
name = normalize_name(library)
case Map.get(@libraries, name) do
nil ->
{:warning, "Unknown library: #{name}"}
_info when profile == :process ->
{:ok, "Process profile isolates workers"}
%{thread_safe: true} ->
{:ok, "Thread-safe"}
%{thread_safe: false, notes: notes} ->
{:error, "Not thread-safe: #{notes}"}
%{thread_safe: :conditional, notes: notes} ->
{:warning, "Conditionally thread-safe: #{notes}"}
end
end
def check(_library, _profile), do: {:error, "Unknown profile"}
@spec get_library_info(String.t() | atom()) :: library_info() | nil
def get_library_info(library) do
mark_legacy_usage()
name = normalize_name(library)
Map.get(@libraries, name)
end
@spec list_all(:thread_safe | :thread_unsafe | :conditional | :all) :: [String.t()]
def list_all(:thread_safe) do
mark_legacy_usage()
select_names(fn {_name, info} -> info.thread_safe == true end)
end
def list_all(:thread_unsafe) do
mark_legacy_usage()
select_names(fn {_name, info} -> info.thread_safe == false end)
end
def list_all(:conditional) do
mark_legacy_usage()
select_names(fn {_name, info} -> info.thread_safe == :conditional end)
end
def list_all(:all) do
mark_legacy_usage()
Map.keys(@libraries)
end
def list_all(_) do
mark_legacy_usage()
[]
end
@spec generate_report([String.t() | atom()], :thread | :process) ::
{:ok, map()} | {:error, term()}
def generate_report(libraries, profile) when is_list(libraries) do
mark_legacy_usage()
report =
Enum.reduce(libraries, %{safe: [], unsafe: [], conditional: [], unknown: []}, fn library,
acc ->
name = normalize_name(library)
case Map.get(@libraries, name) do
nil ->
Map.update!(acc, :unknown, &[name | &1])
%{thread_safe: true} ->
Map.update!(acc, :safe, &[name | &1])
%{thread_safe: false} ->
Map.update!(acc, :unsafe, &[name | &1])
%{thread_safe: :conditional} ->
Map.update!(acc, :conditional, &[name | &1])
end
end)
|> Map.put(:profile, profile)
|> normalize_report()
{:ok, report}
end
def generate_report(_libraries, _profile), do: {:error, :invalid_libraries}
defp normalize_name(library) when is_atom(library),
do: library |> Atom.to_string() |> normalize_name()
defp normalize_name(library) when is_binary(library) do
library
|> String.trim()
|> String.downcase()
|> String.replace("_", "-")
end
defp select_names(fun) do
@libraries
|> Enum.filter(fun)
|> Enum.map(fn {name, _info} -> name end)
end
defp normalize_report(report) do
report
|> Map.update!(:safe, &Enum.reverse/1)
|> Map.update!(:unsafe, &Enum.reverse/1)
|> Map.update!(:conditional, &Enum.reverse/1)
|> Map.update!(:unknown, &Enum.reverse/1)
end
defp mark_legacy_usage do
Deprecation.emit_legacy_module_used(__MODULE__,
replacement: @legacy_replacement,
remove_after: @legacy_remove_after
)
end
end