aistudio-server/catalog.json at master · corespan/aistudio-server · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
{
  "version": "1.1.0",
  "workload_types": [
    {
      "name": "LLMInference",
      "display_name": "LLM Inference (vLLM)",
      "description": "Benchmark LLM inference throughput and latency using vLLM. Measures tokens/sec, TTFT, TPOT, and end-to-end latency across configurable concurrency levels.",
      "image_tag": "1.0.0-nvidia"
    },
    {
      "name": "JupyterNotebook",
      "display_name": "Jupyter Notebook",
      "description": "Interactive Jupyter environment for running microbenchmarks and GPU profiling on remote nodes.",
      "image_tag": "1.0.0-nvidia"
    }
  ],
  "supported_models": [
    {
      "model_id": "tinyllama-1.1b-chat",
      "display_name": "TinyLlama 1.1B Chat",
      "hf_repo": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
      "license": "Apache-2.0",
      "gated": false,
      "license_url": "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0",
      "min_gpu_memory_gb": 4,
      "default_config": {
        "precision": "fp16",
        "gpu_count": 1,
        "max_model_len": 2048,
        "concurrency": 4,
        "input_tokens": 512,
        "output_tokens": 128
      }
    },
    {
      "model_id": "llama3-8b-instruct",
      "display_name": "Llama 3 8B Instruct",
      "hf_repo": "meta-llama/Meta-Llama-3-8B-Instruct",
      "license": "Llama-3-Community",
      "gated": true,
      "license_url": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct",
      "min_gpu_memory_gb": 16,
      "default_config": {
        "precision": "fp16",
        "gpu_count": 1,
        "max_model_len": 4096,
        "concurrency": 4,
        "input_tokens": 512,
        "output_tokens": 512
      }
    },
    {
      "model_id": "llama3-70b-instruct",
      "display_name": "Llama 3 70B Instruct",
      "hf_repo": "meta-llama/Meta-Llama-3-70B-Instruct",
      "license": "Llama-3-Community",
      "gated": true,
      "license_url": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct",
      "min_gpu_memory_gb": 80,
      "default_config": {
        "precision": "fp16",
        "gpu_count": 4,
        "max_model_len": 4096,
        "concurrency": 2,
        "input_tokens": 512,
        "output_tokens": 256
      }
    },
    {
      "model_id": "mistral-7b-instruct",
      "display_name": "Mistral 7B Instruct v0.3",
      "hf_repo": "mistralai/Mistral-7B-Instruct-v0.3",
      "license": "Apache-2.0",
      "gated": false,
      "license_url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3",
      "min_gpu_memory_gb": 16,
      "default_config": {
        "precision": "fp16",
        "gpu_count": 1,
        "max_model_len": 8192,
        "concurrency": 4,
        "input_tokens": 512,
        "output_tokens": 512
      }
    },
    {
      "model_id": "qwen2.5-7b-instruct",
      "display_name": "Qwen 2.5 7B Instruct",
      "hf_repo": "Qwen/Qwen2.5-7B-Instruct",
      "license": "Apache-2.0",
      "gated": false,
      "license_url": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct",
      "min_gpu_memory_gb": 16,
      "default_config": {
        "precision": "fp16",
        "gpu_count": 1,
        "max_model_len": 8192,
        "concurrency": 4,
        "input_tokens": 512,
        "output_tokens": 512
      }
    }
  ],
  "images": {
    "llm_inference": "us-docker.pkg.dev/aimlworkbench/aistudio/llminference",
    "jupyter_notebook": "us-docker.pkg.dev/aimlworkbench/aistudio/jupyternotebook"
  },
  "datasets": {
    "_note": "Datasets are user-supplied. Specify the absolute path to a ShareGPT-format JSON file on the GPU node via dataset_path in the benchmark config. No dataset is downloaded or bundled by this server.",
    "humaneval": {
      "uri": "gs://aistudio-datasets/humaneval.json",
      "license": "MIT",
      "note": "OpenAI HumanEval"
    }
  },
  "_licensing": "Apache-2.0 on this repository covers CoreSpan AI source code only. Models and datasets listed here carry their own licences. See MODEL-LICENSES.md."
}