|
| 1 | +"""Upload MeowLLM artifacts to Hugging Face. |
| 2 | +
|
| 3 | +Usage: |
| 4 | + export HF_TOKEN=your_token_here |
| 5 | + python3 scripts/upload_to_hf.py --username hunt3rx99 |
| 6 | +""" |
| 7 | + |
| 8 | +import argparse |
| 9 | +import os |
| 10 | +import shutil |
| 11 | +import sys |
| 12 | +from pathlib import Path |
| 13 | + |
| 14 | +def main(): |
| 15 | + ap = argparse.ArgumentParser(description="Upload MeowLLM to Hugging Face") |
| 16 | + ap.add_argument("--username", default=os.environ.get("HF_USERNAME"), |
| 17 | + help="HF username (or set HF_USERNAME env var)") |
| 18 | + ap.add_argument("--token", default=os.environ.get("HF_TOKEN"), |
| 19 | + help="HF write token (or set HF_TOKEN env var)") |
| 20 | + args = ap.parse_args() |
| 21 | + |
| 22 | + if not args.username: |
| 23 | + sys.exit("ERROR: pass --username or set HF_USERNAME") |
| 24 | + if not args.token: |
| 25 | + sys.exit("ERROR: pass --token or set HF_TOKEN") |
| 26 | + |
| 27 | + # Check required files |
| 28 | + required = [ |
| 29 | + "checkpoints/best.pt", "data/tokenizer.json", |
| 30 | + "data/train.jsonl", "data/val.jsonl", |
| 31 | + "docs/model_card.md", "docs/dataset_card.md", |
| 32 | + ] |
| 33 | + for f in required: |
| 34 | + if not Path(f).exists(): |
| 35 | + sys.exit(f"ERROR: missing required file: {f}") |
| 36 | + |
| 37 | + from huggingface_hub import HfApi, login |
| 38 | + |
| 39 | + model_repo = f"{args.username}/meowllm" |
| 40 | + dataset_repo = f"{args.username}/meowllm-miso" |
| 41 | + |
| 42 | + print("=" * 42) |
| 43 | + print("MeowLLM Hugging Face Upload") |
| 44 | + print("=" * 42) |
| 45 | + print(f"model repo: {model_repo}") |
| 46 | + print(f"dataset repo: {dataset_repo}") |
| 47 | + print() |
| 48 | + |
| 49 | + # Login |
| 50 | + print("[1/4] Logging in to HF...") |
| 51 | + login(token=args.token) |
| 52 | + api = HfApi() |
| 53 | + |
| 54 | + # Create model repo |
| 55 | + print("\n[2/4] Creating model repo (if needed)...") |
| 56 | + try: |
| 57 | + api.create_repo(model_repo, repo_type="model", exist_ok=True) |
| 58 | + print(f" repo ready: {model_repo}") |
| 59 | + except Exception as e: |
| 60 | + print(f" note: {e}") |
| 61 | + |
| 62 | + # Upload model artifacts |
| 63 | + print("\n[3/4] Uploading model artifacts...") |
| 64 | + api.upload_file(path_or_fileobj="checkpoints/best.pt", |
| 65 | + path_in_repo="best.pt", repo_id=model_repo) |
| 66 | + print(" uploaded best.pt") |
| 67 | + |
| 68 | + api.upload_file(path_or_fileobj="data/tokenizer.json", |
| 69 | + path_in_repo="tokenizer.json", repo_id=model_repo) |
| 70 | + print(" uploaded tokenizer.json") |
| 71 | + |
| 72 | + api.upload_file(path_or_fileobj="docs/model_card.md", |
| 73 | + path_in_repo="README.md", repo_id=model_repo) |
| 74 | + print(" uploaded README.md (model card)") |
| 75 | + |
| 76 | + # Create dataset repo |
| 77 | + print("\n[4/4] Creating and uploading dataset...") |
| 78 | + try: |
| 79 | + api.create_repo(dataset_repo, repo_type="dataset", exist_ok=True) |
| 80 | + print(f" repo ready: {dataset_repo}") |
| 81 | + except Exception as e: |
| 82 | + print(f" note: {e}") |
| 83 | + |
| 84 | + api.upload_file(path_or_fileobj="data/train.jsonl", |
| 85 | + path_in_repo="train.jsonl", |
| 86 | + repo_id=dataset_repo, repo_type="dataset") |
| 87 | + print(" uploaded train.jsonl") |
| 88 | + |
| 89 | + api.upload_file(path_or_fileobj="data/val.jsonl", |
| 90 | + path_in_repo="val.jsonl", |
| 91 | + repo_id=dataset_repo, repo_type="dataset") |
| 92 | + print(" uploaded val.jsonl") |
| 93 | + |
| 94 | + api.upload_file(path_or_fileobj="docs/dataset_card.md", |
| 95 | + path_in_repo="README.md", |
| 96 | + repo_id=dataset_repo, repo_type="dataset") |
| 97 | + print(" uploaded README.md (dataset card)") |
| 98 | + |
| 99 | + print() |
| 100 | + print("=" * 42) |
| 101 | + print("done. check:") |
| 102 | + print(f" https://huggingface.co/{model_repo}") |
| 103 | + print(f" https://huggingface.co/datasets/{dataset_repo}") |
| 104 | + print("=" * 42) |
| 105 | + |
| 106 | + |
| 107 | +if __name__ == "__main__": |
| 108 | + main() |
0 commit comments