Code_mixed/requirements.txt at main · MithraKL/Code_mixed · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# ─────────────────────────────────────────────────────────────────────────────
# requirements.txt
# Code-Mixed Machine Translation with NLLB-200
# ─────────────────────────────────────────────────────────────────────────────

# Core ML
torch>=2.1.0
torchvision>=0.16.0

# HuggingFace Ecosystem
transformers>=4.37.0
datasets>=2.16.0
tokenizers>=0.15.0
accelerate>=0.26.0
evaluate>=0.4.0

# Metrics
sacrebleu>=2.3.1
rouge-score>=0.1.2

# Optional: COMET (neural MT metric)
unbabel-comet>=2.2.0

# Optional: Quantization support
bitsandbytes>=0.42.0

# Optional: ONNX export
optimum[onnxruntime]>=1.17.0

# Optional: DeepSpeed for large-scale training
deepspeed>=0.13.0

# Data & Utilities
pandas>=2.1.0
numpy>=1.26.0
scikit-learn>=1.3.0
tqdm>=4.66.0

# Kaggle API (for downloading LinCE dataset from Kaggle)
kaggle>=1.6.0