Skip to content

Commit 8f90630

Browse files
jorge-martinez-giljorge-martinez-gil
committed
New release
1 parent 9a667c7 commit 8f90630

23 files changed

Lines changed: 2579 additions & 84 deletions

‎.gitignore‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
__pycache__/
2+
*.py[cod]
3+
*.egg-info/
4+
.eggs/
5+
build/
6+
dist/
7+
.pytest_cache/
8+
.venv/
9+
venv/
10+
.DS_Store

‎CITATION.cff‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
cff-version: 1.2.0
2+
title: "UWSD: Context-Aware Semantic Similarity Measurement for Unsupervised Word Sense Disambiguation"
3+
message: "If you use this software or benchmark, please cite the paper below."
4+
type: software
5+
authors:
6+
- family-names: Martinez-Gil
7+
given-names: Jorge
8+
repository-code: "https://github.com/jorge-martinez-gil/uwsd"
9+
license: MIT
10+
keywords:
11+
- word sense disambiguation
12+
- unsupervised WSD
13+
- semantic similarity
14+
- contextual embeddings
15+
- lexical ambiguity
16+
preferred-citation:
17+
type: article
18+
title: "Context-Aware Semantic Similarity Measurement for Unsupervised Word Sense Disambiguation"
19+
authors:
20+
- family-names: Martinez-Gil
21+
given-names: Jorge
22+
year: 2023
23+
journal: "CoRR"
24+
volume: "abs/2305.03520"
25+
url: "https://arxiv.org/abs/2305.03520"
26+
doi: "10.48550/arXiv.2305.03520"

‎CONTRIBUTING.md‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Contributing to UWSD
2+
3+
Thanks for helping make this the standard open benchmark for unsupervised word
4+
sense disambiguation. Contributions of new methods, encoders, datasets, metrics,
5+
tutorials, and bug fixes are all welcome.
6+
7+
## Development setup
8+
9+
```bash
10+
git clone https://github.com/jorge-martinez-gil/uwsd
11+
cd uwsd
12+
pip install -e ".[dev]" # add [bert] / [all] for neural methods
13+
pytest # run the test suite
14+
```
15+
16+
## Adding a new method
17+
18+
Register a class in `uwsd/methods/` and it becomes available from the CLI:
19+
20+
```python
21+
from uwsd.methods import register_method, WSDMethod, Prediction
22+
23+
@register_method("my-method", description="One-line summary.")
24+
class MyMethod(WSDMethod):
25+
def predict(self, instance, task):
26+
scores = {lid: my_score(instance, task, lid) for lid in task.label_ids}
27+
best = max(scores, key=scores.get)
28+
return Prediction(
29+
label_id=best, label=task.classes[best], scores=scores,
30+
confidence=scores[best], explanation="why this sense",
31+
)
32+
```
33+
34+
Guidelines:
35+
36+
- **Interpretability is required.** Populate `scores`, `confidence`, and a short
37+
`explanation` so users can see *why* a sense was chosen.
38+
- **No fabricated results.** Numbers in the README/tables must come from an
39+
actual `uwsd run` manifest. Never hand-edit metrics.
40+
- **Add a test.** At minimum a smoke test that runs your method on one word.
41+
- Keep heavy dependencies optional (add them to an extra in `pyproject.toml`).
42+
43+
## Adding a dataset
44+
45+
Implement a loader returning a `uwsd.data.Dataset` of `WordTask` objects (see
46+
`uwsd/data.py`). Datasets in the CoarseWSD-20 directory layout work out of the
47+
box via `UWSD_DATA`.
48+
49+
## Reporting results
50+
51+
Run with `--keep-correct` so others can re-check significance:
52+
53+
```bash
54+
uwsd run --method my-method --output results/my-method.json --keep-correct
55+
uwsd compare results/my-method.json results/mfs.json
56+
```
57+
58+
## Pull requests
59+
60+
- Run `pytest` and make sure it passes.
61+
- Describe the scientific motivation and include a manifest for any new numbers.
62+
- One focused change per PR where possible.

‎README.md‎

Lines changed: 182 additions & 84 deletions
Large diffs are not rendered by default.

‎pyproject.toml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
[build-system]
2+
requires = ["setuptools>=64", "wheel"]
3+
build-backend = "setuptools.build_meta"
4+
5+
[project]
6+
name = "uwsd"
7+
version = "0.2.0"
8+
description = "Benchmark and experimentation platform for unsupervised word sense disambiguation via context-aware semantic similarity."
9+
readme = "README.md"
10+
license = { text = "MIT" }
11+
requires-python = ">=3.9"
12+
authors = [{ name = "Jorge Martinez-Gil", email = "jorgemarcc@gmail.com" }]
13+
keywords = [
14+
"word sense disambiguation",
15+
"unsupervised WSD",
16+
"semantic similarity",
17+
"contextual embeddings",
18+
"lexical ambiguity",
19+
"WSD benchmark",
20+
"NLP",
21+
"WordNet",
22+
"sentence transformers",
23+
]
24+
classifiers = [
25+
"Programming Language :: Python :: 3",
26+
"License :: OSI Approved :: MIT License",
27+
"Operating System :: OS Independent",
28+
"Intended Audience :: Science/Research",
29+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
30+
"Topic :: Text Processing :: Linguistic",
31+
]
32+
dependencies = [
33+
"numpy>=1.21",
34+
]
35+
36+
[project.optional-dependencies]
37+
bert = ["sentence-transformers>=2.0.0"]
38+
wmd = ["gensim>=4.2.0", "nltk>=3.6"]
39+
use = ["tensorflow-hub>=0.12.0"]
40+
all = ["sentence-transformers>=2.0.0", "gensim>=4.2.0", "nltk>=3.6", "tensorflow-hub>=0.12.0"]
41+
dev = ["pytest>=7.0"]
42+
43+
[project.urls]
44+
Homepage = "https://github.com/jorge-martinez-gil/uwsd"
45+
Paper = "https://arxiv.org/abs/2305.03520"
46+
Issues = "https://github.com/jorge-martinez-gil/uwsd/issues"
47+
48+
[project.scripts]
49+
uwsd = "uwsd.cli:main"
50+
51+
[tool.setuptools]
52+
packages = ["uwsd", "uwsd.methods"]
53+
54+
[tool.setuptools.package-data]
55+
uwsd = ["py.typed"]
56+
57+
[tool.pytest.ini_options]
58+
testpaths = ["tests"]
59+
addopts = "-q"

‎results/hashing.json‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
{
2+
"method": {
3+
"name": "similarity",
4+
"encoder": "hashing-512d-3gram",
5+
"config": {
6+
"model": "all-MiniLM-L6-v2",
7+
"backend": "sentence-transformer"
8+
}
9+
},
10+
"dataset": {
11+
"name": "CoarseWSD-20",
12+
"words": [
13+
"apple",
14+
"arm",
15+
"bank",
16+
"bass",
17+
"bow",
18+
"chair",
19+
"club",
20+
"crane",
21+
"deck",
22+
"digit",
23+
"hood",
24+
"java",
25+
"mole",
26+
"pitcher",
27+
"pound",
28+
"seal",
29+
"spring",
30+
"square",
31+
"trunk",
32+
"yard"
33+
],
34+
"num_test_instances": 10196,
35+
"limit_per_word": null
36+
},
37+
"metrics": {
38+
"micro_accuracy": 0.5504119262455865,
39+
"accuracy_ci95": [
40+
0.5410945468811299,
41+
0.5600259905845429
42+
],
43+
"macro_f1_over_words": 0.27949226914926617,
44+
"weighted_f1_over_words": 0.4735079632019228,
45+
"hits": 5612,
46+
"n": 10196,
47+
"per_word": {
48+
"apple": {
49+
"n": 1032,
50+
"hits": 438,
51+
"accuracy": 0.42441860465116277,
52+
"macro_f1": 0.37205957673475243,
53+
"weighted_f1": 0.33059403784793073,
54+
"num_senses": 2
55+
},
56+
"arm": {
57+
"n": 164,
58+
"hits": 45,
59+
"accuracy": 0.27439024390243905,
60+
"macro_f1": 0.22601626016260162,
61+
"weighted_f1": 0.13398770573071586,
62+
"num_senses": 2
63+
},
64+
"bank": {
65+
"n": 455,
66+
"hits": 22,
67+
"accuracy": 0.04835164835164835,
68+
"macro_f1": 0.04612159329140461,
69+
"weighted_f1": 0.004460110120487478,
70+
"num_senses": 2
71+
},
72+
"bass": {
73+
"n": 1391,
74+
"hits": 88,
75+
"accuracy": 0.06326383896477354,
76+
"macro_f1": 0.03966644128915934,
77+
"weighted_f1": 0.007528354062069063,
78+
"num_senses": 3
79+
},
80+
"bow": {
81+
"n": 215,
82+
"hits": 30,
83+
"accuracy": 0.13953488372093023,
84+
"macro_f1": 0.09603345036415901,
85+
"weighted_f1": 0.061501826013820156,
86+
"num_senses": 3
87+
},
88+
"chair": {
89+
"n": 130,
90+
"hits": 88,
91+
"accuracy": 0.676923076923077,
92+
"macro_f1": 0.4036697247706422,
93+
"weighted_f1": 0.5465067043048695,
94+
"num_senses": 2
95+
},
96+
"club": {
97+
"n": 202,
98+
"hits": 73,
99+
"accuracy": 0.3613861386138614,
100+
"macro_f1": 0.17696969696969697,
101+
"weighted_f1": 0.19186318631863186,
102+
"num_senses": 3
103+
},
104+
"crane": {
105+
"n": 157,
106+
"hits": 84,
107+
"accuracy": 0.535031847133758,
108+
"macro_f1": 0.4022323058467636,
109+
"weighted_f1": 0.4112052478256145,
110+
"num_senses": 2
111+
},
112+
"deck": {
113+
"n": 99,
114+
"hits": 89,
115+
"accuracy": 0.898989898989899,
116+
"macro_f1": 0.4734042553191489,
117+
"weighted_f1": 0.8798624543305394,
118+
"num_senses": 2
119+
},
120+
"digit": {
121+
"n": 42,
122+
"hits": 14,
123+
"accuracy": 0.3333333333333333,
124+
"macro_f1": 0.32723112128146453,
125+
"weighted_f1": 0.2906178489702517,
126+
"num_senses": 2
127+
},
128+
"hood": {
129+
"n": 82,
130+
"hits": 21,
131+
"accuracy": 0.25609756097560976,
132+
"macro_f1": 0.1585329784087548,
133+
"weighted_f1": 0.1400437163200381,
134+
"num_senses": 3
135+
},
136+
"java": {
137+
"n": 1929,
138+
"hits": 1184,
139+
"accuracy": 0.6137895282529808,
140+
"macro_f1": 0.3853442911125225,
141+
"weighted_f1": 0.46906869349988434,
142+
"num_senses": 2
143+
},
144+
"mole": {
145+
"n": 206,
146+
"hits": 43,
147+
"accuracy": 0.2087378640776699,
148+
"macro_f1": 0.1208869179600887,
149+
"weighted_f1": 0.09807977956213806,
150+
"num_senses": 5
151+
},
152+
"pitcher": {
153+
"n": 2819,
154+
"hits": 2806,
155+
"accuracy": 0.9953884356154665,
156+
"macro_f1": 0.4988444444444445,
157+
"weighted_f1": 0.9930879823420441,
158+
"num_senses": 2
159+
},
160+
"pound": {
161+
"n": 97,
162+
"hits": 86,
163+
"accuracy": 0.8865979381443299,
164+
"macro_f1": 0.5465363365915852,
165+
"weighted_f1": 0.8582594713482677,
166+
"num_senses": 2
167+
},
168+
"seal": {
169+
"n": 363,
170+
"hits": 141,
171+
"accuracy": 0.3884297520661157,
172+
"macro_f1": 0.2607110026973238,
173+
"weighted_f1": 0.3025231192018397,
174+
"num_senses": 4
175+
},
176+
"spring": {
177+
"n": 457,
178+
"hits": 150,
179+
"accuracy": 0.3282275711159737,
180+
"macro_f1": 0.18148684371646104,
181+
"weighted_f1": 0.16779880681401488,
182+
"num_senses": 3
183+
},
184+
"square": {
185+
"n": 207,
186+
"hits": 103,
187+
"accuracy": 0.4975845410628019,
188+
"macro_f1": 0.16612903225806452,
189+
"weighted_f1": 0.33065295309334586,
190+
"num_senses": 4
191+
},
192+
"trunk": {
193+
"n": 77,
194+
"hits": 46,
195+
"accuracy": 0.5974025974025974,
196+
"macro_f1": 0.24932249322493225,
197+
"weighted_f1": 0.45655157850279804,
198+
"num_senses": 3
199+
},
200+
"yard": {
201+
"n": 72,
202+
"hits": 61,
203+
"accuracy": 0.8472222222222222,
204+
"macro_f1": 0.45864661654135336,
205+
"weighted_f1": 0.7771512113617376,
206+
"num_senses": 2
207+
}
208+
}
209+
},
210+
"environment": {
211+
"uwsd_version": "0.2.0",
212+
"python": "3.10.12",
213+
"platform": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
214+
"git_commit": "9a667c7",
215+
"numpy_version": "2.2.6"
216+
},
217+
"timestamp_utc": "2026-06-28T13:51:29.362620+00:00"
218+
}

0 commit comments

Comments
 (0)