Skip to content

Commit c9bc41e

Browse files
committed
fix: deps sync optimize
1 parent b66fc68 commit c9bc41e

5 files changed

Lines changed: 82 additions & 94 deletions

File tree

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,5 @@ cmd/ggo/ggo
5353
# .idea/
5454
# .vscode/
5555
.vscode-test/
56-
*.log
56+
*.log
57+
bin/

bin/ggo

-34.2 MB
Binary file not shown.

cmd/ggo/deps/deps.go

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -82,16 +82,23 @@ func newSyncCmd() *cobra.Command {
8282
if manifest != nil {
8383
fmt.Printf("Synced %d libraries (manifest version: %s)\n", len(manifest.Libraries), manifest.Version)
8484

85-
if verbose {
86-
fmt.Println("\nSynced libraries:")
87-
for _, lib := range manifest.Libraries {
88-
fmt.Printf(" Name: %s\n", lib.Name)
89-
fmt.Printf(" Version: %s\n", lib.Version)
90-
fmt.Printf(" Platform: %s/%s\n", lib.Platform, lib.Arch)
91-
fmt.Printf(" Size: %d bytes\n", lib.Size)
92-
fmt.Printf(" SHA256: %s\n", lib.SHA256)
93-
fmt.Printf(" URL: %s\n", lib.URL)
94-
fmt.Println()
85+
if len(manifest.Libraries) > 0 {
86+
if verbose {
87+
fmt.Println("\nSynced libraries:")
88+
for _, lib := range manifest.Libraries {
89+
fmt.Printf(" Name: %s\n", lib.Name)
90+
fmt.Printf(" Version: %s\n", lib.Version)
91+
fmt.Printf(" Platform: %s/%s\n", lib.Platform, lib.Arch)
92+
fmt.Printf(" Size: %d bytes\n", lib.Size)
93+
fmt.Printf(" SHA256: %s\n", lib.SHA256)
94+
fmt.Printf(" URL: %s\n", lib.URL)
95+
fmt.Println()
96+
}
97+
} else {
98+
fmt.Println("\nSynced libraries:")
99+
for _, lib := range manifest.Libraries {
100+
fmt.Printf(" %s (version: %s, platform: %s/%s)\n", lib.Name, lib.Version, lib.Platform, lib.Arch)
101+
}
95102
}
96103
}
97104
}

internal/agent/device.go

Lines changed: 48 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -59,48 +59,33 @@ func DownloadOrFindAccelerator() (string, error) {
5959
// Step 1: Detect vendor (config has highest priority)
6060
vendor, version := detectVendor()
6161
if vendor == "" {
62-
vendor = "STUB" // Fallback to stub if detection fails
62+
vendor = "stub" // Fallback to stub if detection fails (use lowercase for slug matching)
6363
}
64+
// Normalize vendor to lowercase for slug matching
65+
vendorSlug := strings.ToLower(vendor)
6466

65-
// Step 2: Try to find library locally
66-
suffix := getLibSuffix()
67-
libName := fmt.Sprintf("libaccelerator_%s%s", vendor, suffix)
68-
69-
searchPaths := buildSearchPaths()
70-
for _, dir := range searchPaths {
71-
if path := filepath.Join(dir, libName); fileExists(path) {
72-
return path, nil
73-
}
74-
}
75-
76-
// Step 3: Library not found locally, try to download from CDN
67+
// Step 2: Initialize deps manager and fetch manifest
68+
// This will auto-sync on first use if manifest doesn't exist
7769
paths := platform.DefaultPaths()
78-
79-
// Check config for accelerator library version (version already detected from env/config in detectVendor)
80-
// TODO: Add AcceleratorVersion field to config.Config struct for persistent config
81-
82-
// Use deps manager to download
8370
depsMgr := deps.NewManager(deps.WithPaths(paths))
8471
ctx := context.Background()
8572

86-
// Fetch manifest to get latest version or use configured version
73+
// Fetch manifest (auto-syncs if not cached)
8774
manifest, err := depsMgr.FetchManifest(ctx)
8875
if err != nil {
89-
return "", fmt.Errorf("failed to fetch manifest from CDN: %w", err)
76+
return "", fmt.Errorf("failed to fetch manifest: %w", err)
9077
}
9178

92-
// Find library in manifest matching platform and vendor
79+
// Step 3: Find library in manifest matching platform and vendor slug
9380
var targetLib *deps.Library
9481
var candidateLibs []deps.Library
9582
platformLibs := depsMgr.GetLibrariesForPlatform(manifest, "", "")
9683

97-
// Match library name pattern: contains "accelerator_{vendor}"
98-
vendorPattern := "accelerator_" + vendor
84+
// Match by vendor slug from manifest
9985
for i := range platformLibs {
10086
lib := platformLibs[i]
10187
libNameLower := strings.ToLower(lib.Name)
102-
// Match if library name contains the vendor pattern
103-
if strings.Contains(libNameLower, vendorPattern) {
88+
if strings.Contains(libNameLower, "accelerator") && lib.VendorSlug == vendorSlug {
10489
candidateLibs = append(candidateLibs, lib)
10590
}
10691
}
@@ -121,13 +106,47 @@ func DownloadOrFindAccelerator() (string, error) {
121106
}
122107

123108
if targetLib == nil {
124-
// Library not in manifest, return error (could fallback to stub)
109+
// Library not in manifest, return error with available libraries
125110
availableNames := getLibraryNames(platformLibs)
126-
return "", fmt.Errorf("accelerator library for vendor %s (name: %s) not found in CDN manifest. Available libraries: %v",
127-
vendor, libName, availableNames)
111+
return "", fmt.Errorf("accelerator library for vendor %s (slug: %s) not found in CDN manifest. Available libraries: %v",
112+
vendor, vendorSlug, availableNames)
128113
}
129114

130-
// Download the library
115+
// Step 4: Check if library is already downloaded in cache directory
116+
// Search cache directory based on manifest entries
117+
cacheDir := paths.CacheDir()
118+
cachedPath := filepath.Join(cacheDir, targetLib.Name)
119+
if fileExists(cachedPath) {
120+
// Verify the cached file matches the expected hash
121+
if targetLib.SHA256 != "" {
122+
// Use deps manager's VerifyLibrary method
123+
if depsMgr.VerifyLibrary(cachedPath, targetLib.SHA256) {
124+
// Library found in cache, check if installed
125+
installedPath := depsMgr.GetLibraryPath(targetLib.Name)
126+
if fileExists(installedPath) {
127+
return installedPath, nil
128+
}
129+
// Install from cache
130+
if err := depsMgr.InstallLibrary(*targetLib); err != nil {
131+
return "", fmt.Errorf("failed to install library from cache: %w", err)
132+
}
133+
return installedPath, nil
134+
}
135+
} else {
136+
// No hash to verify, assume cached file is valid
137+
installedPath := depsMgr.GetLibraryPath(targetLib.Name)
138+
if fileExists(installedPath) {
139+
return installedPath, nil
140+
}
141+
// Install from cache
142+
if err := depsMgr.InstallLibrary(*targetLib); err != nil {
143+
return "", fmt.Errorf("failed to install library from cache: %w", err)
144+
}
145+
return installedPath, nil
146+
}
147+
}
148+
149+
// Step 5: Library not found locally, download from CDN
131150
progressFn := func(downloaded, total int64) {
132151
// Silent progress for library download
133152
}
@@ -290,50 +309,6 @@ func CreateMockGPUs(count int) []api.GPUInfo {
290309
return gpus
291310
}
292311

293-
func getLibSuffix() string {
294-
switch runtime.GOOS {
295-
case "darwin":
296-
// In provider build, we only build .so libraries, even for MacOS
297-
return ".so"
298-
case "windows":
299-
return ".dll"
300-
default:
301-
return ".so"
302-
}
303-
}
304-
305-
func buildSearchPaths() []string {
306-
paths := []string{
307-
"/usr/lib/tensor-fusion",
308-
"/usr/local/lib/tensor-fusion",
309-
"/opt/tensor-fusion/lib",
310-
}
311-
312-
// Add platform lib directory (where deps manager installs libraries)
313-
platformPaths := platform.DefaultPaths()
314-
paths = append(paths, platformPaths.LibDir())
315-
316-
// Environment variable takes priority
317-
if tfLibPath := os.Getenv("TENSOR_FUSION_LIB_PATH"); tfLibPath != "" {
318-
paths = append([]string{tfLibPath}, paths...)
319-
}
320-
321-
// Add home directory paths
322-
if home, err := os.UserHomeDir(); err == nil {
323-
paths = append(paths,
324-
filepath.Join(home, ".tensor-fusion", "libs"),
325-
filepath.Join(home, ".local", "lib", "tensor-fusion"),
326-
)
327-
}
328-
329-
// Add cwd for development
330-
if cwd, err := os.Getwd(); err == nil {
331-
paths = append(paths, cwd)
332-
}
333-
334-
return paths
335-
}
336-
337312
func fileExists(path string) bool {
338313
_, err := os.Stat(path)
339314
return err == nil

internal/deps/deps.go

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ type Library struct {
4242
URL string `json:"url"`
4343
SHA256 string `json:"sha256"`
4444
Size int64 `json:"size"`
45+
// Vendor information from release
46+
VendorSlug string `json:"vendorSlug,omitempty"` // e.g., "stub", "nvidia", "amd"
47+
VendorName string `json:"vendorName,omitempty"` // e.g., "STUB", "NVIDIA", "AMD"
4548
}
4649

4750
// Manifest represents the version manifest from CDN
@@ -186,13 +189,15 @@ func (m *Manager) SyncReleases(ctx context.Context, os, arch string) (*Manifest,
186189
}
187190

188191
lib := Library{
189-
Name: libName,
190-
Version: release.Version,
191-
Platform: artifactOS,
192-
Arch: artifactArch,
193-
URL: artifact.URL,
194-
SHA256: artifact.SHA256,
195-
Size: size,
192+
Name: libName,
193+
Version: release.Version,
194+
Platform: artifactOS,
195+
Arch: artifactArch,
196+
URL: artifact.URL,
197+
SHA256: artifact.SHA256,
198+
Size: size,
199+
VendorSlug: strings.ToLower(release.Vendor.Slug),
200+
VendorName: release.Vendor.Name,
196201
}
197202
manifest.Libraries = append(manifest.Libraries, lib)
198203
}
@@ -363,7 +368,7 @@ func (m *Manager) DownloadLibrary(ctx context.Context, lib Library, progressFn f
363368
tmpPath := destPath + ".tmp"
364369

365370
// Check if already downloaded with correct hash (skip if SHA256 is empty)
366-
if lib.SHA256 != "" && m.verifyLibrary(destPath, lib.SHA256) {
371+
if lib.SHA256 != "" && m.VerifyLibrary(destPath, lib.SHA256) {
367372
return nil // Already downloaded
368373
}
369374

@@ -539,9 +544,9 @@ func (m *Manager) CleanCache() error {
539544
return nil
540545
}
541546

542-
// verifyLibrary checks if a file exists and has the expected hash
547+
// VerifyLibrary checks if a file exists and has the expected hash
543548
// Returns true if expectedHash is empty (verification skipped)
544-
func (m *Manager) verifyLibrary(path, expectedHash string) bool {
549+
func (m *Manager) VerifyLibrary(path, expectedHash string) bool {
545550
// Skip verification if hash is empty
546551
if expectedHash == "" {
547552
// Just check if file exists

0 commit comments

Comments
 (0)