Skip to content

Commit 972d698

Browse files
yizheliu-amazonyizheliu-amazon
authored
[Enhancement] Add model_selection parameter to semantic field to resolve model id from cluster settings (#1919)
* Add model_selection parameter to semantic field to resolve model id from cluster settings (#1918) Introduces a `model_selection` nested object (language_option + model_type) on the semantic field. The model id is resolved from operator-configured cluster settings `plugins.neural_search.model_selection.model_id.<model_type>.<language_option>` (an affix setting, flexible for future dimensions). No model is auto-deployed and there is no cache: the operator deploys the model and configures the setting. - ModelSelection DTO validates language_option (ENGLISH|MULTILINGUAL) and model_type (SPARSE|DENSE). - ClusterSettingSemanticModelResolver reads the configured model id, fails with a clear error when unset, and verifies the model exists and its type matches the requested model_type. - SemanticMappingTransformer resolves model_selection fields; when both model_id and model_selection are provided they must resolve to the same model id, otherwise the request is rejected. Signed-off-by: Yizhe Liu <yizheliu@amazon.com> * [temporary] CI: resolve Eclipse JDT formatter without ci.opensearch.org P2 mirror Same change as #1964, applied here so #1919 CI can run past the eclipse-jdt SocketTimeoutException that the ci.opensearch.org P2 mirror (added in #1940) is currently throwing at configuration time. Revert once #1964 lands on main. See opensearch-project/OpenSearch#20826. Signed-off-by: Yizhe Liu <yizheliu@amazon.com> --------- Signed-off-by: Yizhe Liu <yizheliu@amazon.com>
1 parent 6c46775 commit 972d698

21 files changed

Lines changed: 1581 additions & 14 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
88
### Features
99

1010
### Enhancements
11+
- Add `model_selection` (language_option/model_type) parameter to semantic field to resolve the model id from cluster settings ([#1918](https://github.com/opensearch-project/neural-search/issues/1918))
1112

1213
### Bug Fixes
1314
* [Hybrid Query] Fix NoSuchElementException in hybrid query with sort/search_after when a shard returns no results ([#1939](https://github.com/opensearch-project/neural-search/pull/1939))

‎formatter/formatting.gradle‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@ allprojects {
77
target '**/*.java'
88

99
removeUnusedImports()
10-
eclipse().withP2Mirrors(Map.of("https://download.eclipse.org/", "https://ci.opensearch.org/")).configFile rootProject.file('formatter/formatterConfig.xml')
10+
// Resolve the Eclipse JDT formatter directly from download.eclipse.org. The prior
11+
// withP2Mirrors(... -> https://ci.opensearch.org/) mirror (added in #1940) started timing
12+
// out at configuration time (SocketTimeoutException), failing every task on affected
13+
// runners. This aligns with the OpenSearch 3.x resolution in opensearch-project/OpenSearch#20826.
14+
eclipse().configFile rootProject.file('formatter/formatterConfig.xml')
1115
trimTrailingWhitespace()
1216
endWithNewline();
1317

‎src/main/java/org/opensearch/neuralsearch/constants/SemanticFieldConstants.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,23 @@ public class SemanticFieldConstants {
6666
* model is not changed.
6767
*/
6868
public static final String SKIP_EXISTING_EMBEDDING = "skip_existing_embedding";
69+
70+
/**
71+
* Name of the model selection parameter. It is a nested object holding {@link #LANGUAGE_OPTION} and
72+
* {@link #MODEL_TYPE}. When specified, the system resolves the {@code model_id} from the
73+
* {@code plugins.neural_search.model_selection.model_id.*} cluster settings instead of requiring the customer to
74+
* provide a {@code model_id} directly.
75+
*/
76+
public static final String MODEL_SELECTION = "model_selection";
77+
78+
/**
79+
* Name of the language option sub-parameter of {@link #MODEL_SELECTION}. Supported values: ENGLISH, MULTILINGUAL.
80+
*/
81+
public static final String LANGUAGE_OPTION = "language_option";
82+
83+
/**
84+
* Name of the model type sub-parameter of {@link #MODEL_SELECTION}. Used together with {@link #LANGUAGE_OPTION} to
85+
* resolve the appropriate model. Supported values: SPARSE, DENSE.
86+
*/
87+
public static final String MODEL_TYPE = "model_type";
6988
}

‎src/main/java/org/opensearch/neuralsearch/mapper/SemanticFieldMapper.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.opensearch.index.mapper.WildcardFieldMapper;
2323
import org.opensearch.neuralsearch.constants.MappingConstants;
2424
import org.opensearch.neuralsearch.mapper.dto.ChunkingConfig;
25+
import org.opensearch.neuralsearch.mapper.dto.ModelSelection;
2526
import org.opensearch.neuralsearch.mapper.dto.SemanticParameters;
2627
import org.opensearch.neuralsearch.processor.chunker.ChunkerValidatorFactory;
2728
import org.opensearch.neuralsearch.mapper.dto.SparseEncodingConfig;
@@ -39,6 +40,7 @@
3940
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.DEFAULT_SEMANTIC_INFO_FIELD_NAME_SUFFIX;
4041
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.DENSE_EMBEDDING_CONFIG;
4142
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.MODEL_ID;
43+
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.MODEL_SELECTION;
4244
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.RAW_FIELD_TYPE;
4345
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.SKIP_EXISTING_EMBEDDING;
4446
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.SEARCH_MODEL_ID;
@@ -211,6 +213,21 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
211213
false
212214
);
213215

216+
@Getter
217+
protected final Parameter<ModelSelection> modelSelection = new Parameter<>(
218+
MODEL_SELECTION,
219+
true,
220+
() -> null,
221+
(name, ctx, value) -> value == null ? null : new ModelSelection(name, value),
222+
m -> ((SemanticFieldMapper) m).semanticParameters.getModelSelection()
223+
).setSerializer((builder, name, value) -> {
224+
if (value == null) {
225+
builder.nullField(name);
226+
} else {
227+
value.toXContent(builder, name);
228+
}
229+
}, (value) -> value == null ? null : value.toString());
230+
214231
@Setter
215232
protected ParametrizedFieldMapper.Builder delegateBuilder;
216233

@@ -229,7 +246,8 @@ protected List<Parameter<?>> getParameters() {
229246
semanticFieldSearchAnalyzer,
230247
denseEmbeddingConfig,
231248
sparseEncodingConfig,
232-
skipExistingEmbedding
249+
skipExistingEmbedding,
250+
modelSelection
233251
);
234252
}
235253

@@ -261,6 +279,7 @@ public SemanticParameters getSemanticParameters() {
261279
.denseEmbeddingConfig(denseEmbeddingConfig.getValue())
262280
.sparseEncodingConfig(sparseEncodingConfig.getValue())
263281
.skipExistingEmbedding(skipExistingEmbedding.getValue())
282+
.modelSelection(modelSelection.getValue())
264283
.build();
265284
}
266285
}

‎src/main/java/org/opensearch/neuralsearch/mapper/dto/ModelSelection.java‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
package org.opensearch.neuralsearch.mapper.dto;
6+
7+
import lombok.Getter;
8+
import lombok.NonNull;
9+
import org.apache.commons.lang3.builder.EqualsBuilder;
10+
import org.apache.commons.lang3.builder.HashCodeBuilder;
11+
import org.opensearch.core.xcontent.XContentBuilder;
12+
import org.opensearch.index.mapper.MapperParsingException;
13+
14+
import java.io.IOException;
15+
import java.util.Locale;
16+
import java.util.Map;
17+
import java.util.Set;
18+
19+
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.LANGUAGE_OPTION;
20+
import static org.opensearch.neuralsearch.constants.SemanticFieldConstants.MODEL_TYPE;
21+
22+
/**
23+
* DTO for the {@code model_selection} nested object of a semantic field. It holds the human readable
24+
* description of the model the customer wants (language + model type) which the system resolves to a
25+
* concrete {@code model_id} through the {@code plugins.neural_search.model_selection.model_id.*} cluster
26+
* settings.
27+
* <p>
28+
* Example:
29+
* <pre>
30+
* "model_selection": {
31+
* "language_option": "ENGLISH",
32+
* "model_type": "SPARSE"
33+
* }
34+
* </pre>
35+
*/
36+
@Getter
37+
public class ModelSelection {
38+
public static final String ENGLISH = "ENGLISH";
39+
public static final String MULTILINGUAL = "MULTILINGUAL";
40+
public static final String SPARSE = "SPARSE";
41+
public static final String DENSE = "DENSE";
42+
43+
public static final Set<String> SUPPORTED_LANGUAGE_OPTIONS = Set.of(ENGLISH, MULTILINGUAL);
44+
public static final Set<String> SUPPORTED_MODEL_TYPES = Set.of(SPARSE, DENSE);
45+
46+
private final String languageOption;
47+
private final String modelType;
48+
49+
/**
50+
* Construct a ModelSelection from the raw value defined under {@code model_selection} in the index mappings.
51+
* Missing sub-fields default to ENGLISH / SPARSE.
52+
*
53+
* @param name parameter name (used for error messages)
54+
* @param value raw parameter value, expected to be a Map
55+
*/
56+
@SuppressWarnings("unchecked")
57+
public ModelSelection(@NonNull final String name, final Object value) {
58+
if (value instanceof Map == false) {
59+
throw new MapperParsingException(String.format(Locale.ROOT, "[%s] must be a Map", name));
60+
}
61+
final Map<String, Object> config = (Map<String, Object>) value;
62+
63+
for (final String key : config.keySet()) {
64+
if (LANGUAGE_OPTION.equals(key) == false && MODEL_TYPE.equals(key) == false) {
65+
throw new MapperParsingException(String.format(Locale.ROOT, "Unsupported parameter [%s] in [%s]", key, name));
66+
}
67+
}
68+
69+
this.languageOption = normalize(config.get(LANGUAGE_OPTION), LANGUAGE_OPTION, ENGLISH, SUPPORTED_LANGUAGE_OPTIONS);
70+
this.modelType = normalize(config.get(MODEL_TYPE), MODEL_TYPE, SPARSE, SUPPORTED_MODEL_TYPES);
71+
}
72+
73+
/**
74+
* Construct a ModelSelection directly from its language option and model type.
75+
*/
76+
public ModelSelection(final String languageOption, final String modelType) {
77+
this.languageOption = normalize(languageOption, LANGUAGE_OPTION, ENGLISH, SUPPORTED_LANGUAGE_OPTIONS);
78+
this.modelType = normalize(modelType, MODEL_TYPE, SPARSE, SUPPORTED_MODEL_TYPES);
79+
}
80+
81+
private static String normalize(final Object rawValue, final String field, final String defaultValue, final Set<String> supported) {
82+
if (rawValue == null) {
83+
return defaultValue;
84+
}
85+
final String normalized = rawValue.toString().toUpperCase(Locale.ROOT);
86+
if (supported.contains(normalized) == false) {
87+
throw new MapperParsingException(
88+
String.format(
89+
Locale.ROOT,
90+
"Unsupported [%s] value [%s]. It should be one of [%s].",
91+
field,
92+
rawValue,
93+
String.join(", ", supported)
94+
)
95+
);
96+
}
97+
return normalized;
98+
}
99+
100+
/**
101+
* @return whether the customer requested a dense model.
102+
*/
103+
public boolean isDense() {
104+
return DENSE.equals(modelType);
105+
}
106+
107+
/**
108+
* @return the profile key used to look up the resolved model_id from the cluster settings. It is the suffix of the
109+
* affix setting {@code plugins.neural_search.model_selection.model_id.} and takes the form {@code <model_type>.<language_option>}
110+
* (both lower cased), e.g. {@code sparse.english}.
111+
*/
112+
public String getProfileKey() {
113+
return modelType.toLowerCase(Locale.ROOT) + "." + languageOption.toLowerCase(Locale.ROOT);
114+
}
115+
116+
public void toXContent(@NonNull final XContentBuilder builder, final String name) throws IOException {
117+
builder.startObject(name);
118+
builder.field(LANGUAGE_OPTION, languageOption);
119+
builder.field(MODEL_TYPE, modelType);
120+
builder.endObject();
121+
}
122+
123+
@Override
124+
public String toString() {
125+
return String.format(Locale.ROOT, "{%s=%s, %s=%s}", LANGUAGE_OPTION, languageOption, MODEL_TYPE, modelType);
126+
}
127+
128+
@Override
129+
public boolean equals(final Object obj) {
130+
if (this == obj) {
131+
return true;
132+
} else if (obj != null && this.getClass() == obj.getClass()) {
133+
final ModelSelection other = (ModelSelection) obj;
134+
return new EqualsBuilder().append(this.languageOption, other.languageOption).append(this.modelType, other.modelType).isEquals();
135+
} else {
136+
return false;
137+
}
138+
}
139+
140+
@Override
141+
public int hashCode() {
142+
return new HashCodeBuilder().append(this.languageOption).append(this.modelType).toHashCode();
143+
}
144+
}

‎src/main/java/org/opensearch/neuralsearch/mapper/dto/SemanticParameters.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ public class SemanticParameters {
2323
private final String semanticFieldSearchAnalyzer;
2424
private final Map<String, Object> denseEmbeddingConfig;
2525
private final SparseEncodingConfig sparseEncodingConfig;
26+
private final ModelSelection modelSelection;
2627

2728
public boolean isChunkingEnabled() {
2829
if (chunkingConfig == null) {

0 commit comments

Comments
 (0)