package download import ( "bufio" "crypto/md5" "fmt" "io" "net/http" "os" "path/filepath" "strings" "gorm.io/gorm" "rhemawavetech.com/rhemawave/internal/db" "rhemawavetech.com/rhemawave/internal/logger" ) // DownloadStatus holds information related to downloads and is // the container for information returned via channels within this package. type DownloadStatus struct { Max int64 Cur int64 Msg string Err error } // manifest maps filename → expected MD5 hex string type manifest map[string]string const manifestFilename = "checksums.md5" // DownloadAudioFiles downloads audio files using the remote checksums.md5 as // the source of truth. It: // - Downloads any file in the manifest that is missing locally. // - Re-downloads files whose local MD5 differs from the manifest. // - Removes local files not listed in the manifest (orphans). // - Removes DB AudioTrack records not listed in the manifest. // - Updates AudioTrack.MD5 in the database after every successful download. // // Progress is sent to progressChan, which is closed on return. func DownloadAudioFiles(progressChan chan<- DownloadStatus, dbHandle *gorm.DB, destDir string, urlBase string) { defer close(progressChan) var totalDownloaded int64 var filesDownloaded int send := func(cur int64, msg string, err error) { progressChan <- DownloadStatus{Cur: cur, Max: 0, Msg: msg, Err: err} } send(0, "Fetching file manifest...", nil) m, err := fetchManifest(urlBase + manifestFilename) if err != nil { send(0, "Failed to fetch manifest", err) return } logger.Info("download: manifest loaded, %d files listed", len(m)) send(0, "Comparing checksums...", nil) type work struct { filename string expectedMD5 string } var queue []work for filename, expectedMD5 := range m { localPath := filepath.Join(destDir, filename) localMD5, err := fileMD5(localPath) if err != nil { logger.Debug("download: %s not found locally, queuing", filename) queue = append(queue, work{filename, expectedMD5}) continue } if localMD5 != expectedMD5 { logger.Debug("download: %s checksum mismatch (got %s, want %s), re-downloading", filename, localMD5, expectedMD5) if err := os.Remove(localPath); err != nil { logger.Warning("download: failed to remove corrupt file %s: %v", localPath, err) } queue = append(queue, work{filename, expectedMD5}) } else { logger.Debug("download: %s OK", filename) } } // ── Remove local files not in manifest (orphans) ───────────────────── entries, err := os.ReadDir(destDir) if err == nil { for _, entry := range entries { if entry.IsDir() { continue } if _, ok := m[entry.Name()]; !ok { orphanPath := filepath.Join(destDir, entry.Name()) logger.Info("download: removing orphan file %s", orphanPath) if err := os.Remove(orphanPath); err != nil { logger.Warning("download: failed to remove orphan %s: %v", orphanPath, err) } } } } else { logger.Warning("download: could not read destDir for orphan cleanup: %v", err) } // ── Remove DB records not in manifest ──────────────────────────────── if dbHandle != nil { var allTracks []db.AudioTrack dbHandle.Find(&allTracks) for _, track := range allTracks { if _, ok := m[track.Filename]; !ok { logger.Info("download: removing orphan DB record %s (filename=%s)", track.Name, track.Filename) dbHandle.Delete(&track) } } } if len(queue) == 0 { send(0, "All files are up to date.", nil) logger.Info("download: nothing to do") return } total := len(queue) send(0, fmt.Sprintf("0 / %d Files Downloaded", total), nil) logger.Info("download: %d file(s) to download", total) for _, w := range queue { url := urlBase + w.filename destPath := filepath.Join(destDir, w.filename) logger.Debug("download: starting %s", w.filename) downloaded, err := downloadFile(url, destPath, func(n int64) { totalDownloaded += n send(totalDownloaded, fmt.Sprintf("%d / %d Files Downloaded", filesDownloaded, total), nil) }) if err != nil { logger.Error("download: failed to download %s: %v", w.filename, err) send(totalDownloaded, fmt.Sprintf("%d / %d Files Downloaded", filesDownloaded, total), err) return } gotMD5, err := fileMD5(destPath) if err != nil { logger.Error("download: could not hash %s after download: %v", w.filename, err) os.Remove(destPath) send(totalDownloaded, "Checksum verification failed", err) return } if gotMD5 != w.expectedMD5 { err := fmt.Errorf("checksum mismatch after download: got %s, want %s", gotMD5, w.expectedMD5) logger.Error("download: %s %v", w.filename, err) os.Remove(destPath) send(totalDownloaded, "Checksum verification failed — file removed", err) return } // Update MD5 in database if dbHandle != nil { var track db.AudioTrack result := dbHandle.Where("filename = ?", w.filename).First(&track) if result.Error == nil { track.MD5 = gotMD5 dbHandle.Save(&track) logger.Debug("download: updated MD5 for %s", w.filename) } } filesDownloaded++ statusMsg := fmt.Sprintf("%d / %d Files Downloaded", filesDownloaded, total) logger.Info("download: %s (%d bytes)", statusMsg, downloaded) send(totalDownloaded, statusMsg, nil) } send(totalDownloaded, "All downloads complete!", nil) logger.Info("download: all files downloaded and verified") } // ─── Helpers ────────────────────────────────────────────────────────────────── // fetchManifest downloads and parses a checksums.md5 file. // Format per line: " " (two spaces, same as md5sum output) func fetchManifest(url string) (manifest, error) { resp, err := http.Get(url) if err != nil { return nil, fmt.Errorf("fetch manifest: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("fetch manifest: server returned %s", resp.Status) } m := make(manifest) scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } // md5sum format: "hash filename" (two spaces) parts := strings.SplitN(line, " ", 2) if len(parts) != 2 { logger.Warning("download: skipping malformed manifest line: %q", line) continue } hash := strings.TrimSpace(parts[0]) name := strings.TrimSpace(parts[1]) name = strings.TrimPrefix(name, "./") m[name] = hash } if err := scanner.Err(); err != nil { return nil, fmt.Errorf("parse manifest: %w", err) } return m, nil } // fileMD5 computes the MD5 checksum of a local file. // Returns an error if the file doesn't exist or can't be read. func fileMD5(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer f.Close() h := md5.New() if _, err := io.Copy(h, f); err != nil { return "", fmt.Errorf("hash %s: %w", path, err) } return fmt.Sprintf("%x", h.Sum(nil)), nil } // downloadFile streams url to destPath, calling onChunk(n) after each write // with the number of bytes just written. Returns total bytes downloaded. func downloadFile(url, destPath string, onChunk func(int64)) (int64, error) { resp, err := http.Get(url) if err != nil { return 0, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return 0, fmt.Errorf("GET %s: %s", url, resp.Status) } out, err := os.Create(destPath) if err != nil { return 0, fmt.Errorf("create %s: %w", destPath, err) } defer out.Close() var total int64 buf := make([]byte, 32*1024) // 32KB chunks for { n, err := resp.Body.Read(buf) if n > 0 { if _, werr := out.Write(buf[:n]); werr != nil { os.Remove(destPath) return total, fmt.Errorf("write %s: %w", destPath, werr) } total += int64(n) onChunk(int64(n)) } if err == io.EOF { break } if err != nil { os.Remove(destPath) return total, fmt.Errorf("read %s: %w", url, err) } } return total, nil }