scripts\fetch-libs.ps1

源码路径:scripts\fetch-libs.ps1 · 233 行

# 下载 libs 组件:llama.cpp / stable-diffusion.cpp 二进制 + Qwen-Image 2.1 模型。
#
# 背景:这台机器没有 NVIDIA 卡,是 **Intel Arc B370** —— 所以走
#   llama.cpp            → SYCL(Intel oneAPI,Arc 上最快)+ CPU 兜底
#   stable-diffusion.cpp → Vulkan(29MB,Arc 上能用)+ CPU 兜底
# 没下 CUDA/ROCm 那些几百 MB 的包。
#
# 模型全部从 **ModelScope 国内镜像**拉(HuggingFace 在这台机器上不通)。
#
# Qwen-Image 2.1 需要三个文件,少一个都跑不起来(依据 sd.cpp 官方文档
# docs/qwen_image_2.1.md):
#   1. diffusion model —— 去噪主干
#   2. text encoder     —— **Qwen3-VL-8B**,不是 Qwen2.5(容易搞错)
#   3. VAE              —— 必须是 qwen_image_2.1 专用的;官方明确说
#                          「旧的 Qwen Image / Wan 2.2 VAE 权重不可互换」
# 图片编辑另外还要 mmproj(视觉投影层)。
#
# 编程处理用的是 **Qwen3-Coder-30B-A3B-Instruct**(MoE:30B 总参 / 3B 激活)的
# 4-bit 量化版。官方没出 GGUF,用的是 ModelScope 上 unsloth 的转换。
# 注意本机只有 31.5 GB 内存,17.3 GB 的权重装得下但很紧 ——
# 真拉不起来就换更小的量化(UD-Q4_K_XL 16.8GB / IQ4_XS 15.6GB)。
#
# 用法:
#   .\scripts\fetch-libs.ps1                      # 全部(binaries + 图像模型 + 编程模型)
#   .\scripts\fetch-libs.ps1 -BinariesOnly        # 只下二进制(约 205 MB)
#   .\scripts\fetch-libs.ps1 -ModelsOnly          # 只下模型
#   .\scripts\fetch-libs.ps1 -ImageOnly           # 只下图像模型(约 10.6 GB)
#   .\scripts\fetch-libs.ps1 -CodeOnly            # 只下编程模型(约 17.3 GB)
#   .\scripts\fetch-libs.ps1 -Root C:\somewhere   # 换个程序根
#   .\scripts\fetch-libs.ps1 -SkipVision          # 不下 mmproj(省 1.1GB,放弃图片编辑)
#
# 中断了直接重跑:已下完的会跳过,没下完的断点续传。

[CmdletBinding()]
param(
    [string]$Root = '',
    [switch]$BinariesOnly,
    [switch]$ModelsOnly,
    [switch]$SkipVision,
    [switch]$CodeOnly,      # 只下编程模型(Qwen3-Coder-30B)
    [switch]$ImageOnly      # 只下图像模型(Qwen-Image 2.1 那四个文件)
)

$ErrorActionPreference = 'Stop'
$ws = Split-Path -Parent $PSScriptRoot
if (-not $Root) { $Root = Join-Path $ws 'dist' }
if (-not (Test-Path $Root)) { throw "程序根不存在:$Root(先跑 build-release.ps1)" }

$Curl = Join-Path $env:WINDIR 'system32\curl.exe'
$Tar = Join-Path $env:WINDIR 'system32\tar.exe'
if (-not (Test-Path $Curl)) { throw '找不到 curl.exe' }

$libs = Join-Path $Root 'libs'
$dirLlama = Join-Path $libs 'llama.cpp'
$dirSd = Join-Path $libs 'stable-diffusion.cpp'
$dirModel = Join-Path $libs 'model'
$dirFfmpeg = Join-Path $libs 'ffmpeg'
$dirTemp = Join-Path $libs '_download'          # 压缩包临时区,解压后删掉
foreach ($d in $dirLlama, $dirSd, $dirModel, $dirFfmpeg, $dirTemp) { New-Item -ItemType Directory -Force -Path $d | Out-Null }

# ---------------------------------------------------------------- 下载原语

function Get-RemoteSize {
    param([string]$Url)
    # curl 往 stderr 写东西;PowerShell 5.1 在 ErrorActionPreference='Stop' 下
    # 会把它升级成终止性异常 —— 所以调 curl 一律临时放宽。
    $prev = $ErrorActionPreference
    $ErrorActionPreference = 'Continue'
    try {
        $h = @(& $Curl -sIL --max-time 40 $Url 2>&1)
    }
    finally { $ErrorActionPreference = $prev }

    $lens = $h | Select-String -Pattern 'content-length:\s*(\d+)' -AllMatches |
        ForEach-Object { [int64]$_.Matches[0].Groups[1].Value }
    if ($lens.Count -gt 0) { return $lens[$lens.Count - 1] }
    return 0
}

function Fetch-File {
    param(
        [string]$Url,          # 多个源用 | 分隔,按顺序回退
        [string]$Dest,
        [string]$What = ''
    )
    New-Item -ItemType Directory -Force -Path (Split-Path -Parent $Dest) | Out-Null
    $name = Split-Path $Dest -Leaf
    $urls = @($Url.Split('|') | Where-Object { $_.Trim() } | ForEach-Object { $_.Trim() })

    # 远端真实字节数 —— 不要用 API 里四舍五入过的 MB,那样永远判「未完成」(踩过)
    $expect = 0
    foreach ($u in $urls) { $expect = Get-RemoteSize $u; if ($expect -gt 0) { break } }

    if ((Test-Path $Dest) -and $expect -gt 0 -and (Get-Item $Dest).Length -eq $expect) {
        Write-Host ("  [跳过] {0}  已完整 {1:N0} MB" -f $name, ($expect / 1MB)) -ForegroundColor DarkGray
        return $true
    }

    foreach ($u in $urls) {
        for ($i = 1; $i -le 4; $i++) {
            $before = if (Test-Path $Dest) { (Get-Item $Dest).Length } else { 0 }
            $prev = $ErrorActionPreference
            $ErrorActionPreference = 'Continue'
            try {
                # -C - 断点续传;--retry 应对瞬时断流
                & $Curl -sS -L --fail --retry 2 --retry-delay 3 --retry-all-errors `
                    --connect-timeout 30 -C - -o $Dest $u 2>&1 |
                    ForEach-Object { Write-Host "      $_" -ForegroundColor DarkYellow }
                $code = $LASTEXITCODE
            }
            finally { $ErrorActionPreference = $prev }

            $after = if (Test-Path $Dest) { (Get-Item $Dest).Length } else { 0 }
            # 有准确大小就按大小判;拿不到大小(无 content-length)就认 curl 的成功退出码
            $done = if ($expect -gt 0) { $after -eq $expect } else { $code -eq 0 -and $after -gt 0 }
            if ($done) {
                Write-Host ("  [完成] {0}  {1:N0} MB" -f $name, ($after / 1MB)) -ForegroundColor Green
                return $true
            }
            Write-Host ("      第 {0} 次:{1:N0} -> {2:N0} MB (curl={3})" -f $i, ($before / 1MB), ($after / 1MB), $code) -ForegroundColor DarkYellow
            Start-Sleep -Seconds 3
        }
    }
    Write-Host ("  [失败] {0}" -f $What) -ForegroundColor Red
    return $false
}

# ---------------------------------------------------------------- 二进制

$GH = 'https://ghfast.top/https://github.com'
$GHD = 'https://github.com'

$binaries = @(
    @{ Zip = 'llama-b11284-bin-win-sycl-x64.zip';        Dir = $dirLlama; Repo = 'ggml-org/llama.cpp/releases/download/b11284';                    Tag = 'b11284 sycl(Intel Arc)';              What = 'llama.cpp SYCL(Intel Arc)' }
    @{ Zip = 'llama-b11284-bin-win-cpu-x64.zip';         Dir = $dirLlama; Repo = 'ggml-org/llama.cpp/releases/download/b11284';                    Tag = 'b11284 cpu';                          What = 'llama.cpp CPU 兜底' }
    @{ Zip = 'sd-master-3f8527a-bin-win-vulkan-x64.zip'; Dir = $dirSd;    Repo = 'leejet/stable-diffusion.cpp/releases/download/master-929-3f8527a'; Tag = 'master-929-3f8527a vulkan(Intel Arc)'; What = 'sd.cpp Vulkan(Intel Arc)' }
    @{ Zip = 'sd-master-3f8527a-bin-win-cpu-x64.zip';    Dir = $dirSd;    Repo = 'leejet/stable-diffusion.cpp/releases/download/master-929-3f8527a'; Tag = 'master-929-3f8527a cpu';               What = 'sd.cpp CPU 兜底' }
    # ffmpeg:视频抽帧与取时长,播放器和「拉流与解析」都要它。
    # 选静态 GPL 版(191 MB)而不是 shared 版(84.7 MB):shared 版把编解码器放在
    # 一堆 DLL 里,散一个就报"找不到入口点";静态版是三个独立 exe。
    # Strip = 2 是必须的:这个 zip 内层是 ffmpeg-master-latest-win64-gpl/bin/,
    # 不剥路径的话 exe 会埋在子目录里 —— 而程序按 libs\ffmpeg\ffmpeg.exe 找,
    # 找不到就是**静默没有视频**。
    # (上面几行刻意不用反引号:在 PowerShell 里反引号是转义符,注释里也会出问题。)
    @{ Zip = 'ffmpeg-master-latest-win64-gpl.zip';       Dir = $dirFfmpeg; Repo = 'BtbN/FFmpeg-Builds/releases/download/latest';                      Tag = 'master-latest gpl 静态';              What = 'ffmpeg 视频抽帧(约 191 MB)'; Strip = 2 }
)

# -ImageOnly / -CodeOnly 是「只要模型」,不该顺带重下二进制
if (-not ($ModelsOnly -or $ImageOnly -or $CodeOnly)) {
    Write-Host ''
    Write-Host '=== 二进制(约 205 MB)===' -ForegroundColor Cyan
    foreach ($b in $binaries) {
        Write-Host ("- {0}" -f $b.What)
        $tmp = Join-Path $dirTemp $b.Zip
        $url = "$GH/$($b.Repo)/$($b.Zip)|$GHD/$($b.Repo)/$($b.Zip)"
        if (Fetch-File -Url $url -Dest $tmp -What $b.What) {
            # 系统自带的 tar.exe 是 bsdtar,直接支持 zip —— 不必为此引第三方解压库。
            # `Strip` 给内部有嵌套目录的包用(ffmpeg 就是),把多余层级剥掉。
            $stripArgs = if ($b.Strip) { @('--strip-components', "$($b.Strip)") } else { @() }
            & $Tar -xf $tmp -C $b.Dir @stripArgs
            if ($LASTEXITCODE -ne 0) { Write-Host "  [!] 解压失败:$($b.Zip)" -ForegroundColor Yellow }
            else {
                Write-Host ("  [解压] -> {0}" -f (Split-Path $b.Dir -Leaf)) -ForegroundColor DarkGray
                # 记「装的是哪个构建」:界面和 --libs-status 都读它。
                # SYCL 与 CPU 两个包解压到**同一个目录**(DLL 共存),所以这里要
                # 累加去重,不能直接覆盖 —— 否则标记只剩下最后解压的那个,
                # 看起来像"只装了 CPU 版"。
                $marker = Join-Path $b.Dir '.nfarms-version'
                $prev = if (Test-Path $marker) { (Get-Content $marker -Raw).Trim() } else { '' }
                $tags = @($prev -split '\s*\+\s*' | Where-Object { $_ }) + $b.Tag
                Set-Content $marker ((@($tags | Select-Object -Unique)) -join ' + ') -Encoding ASCII
                Remove-Item $tmp -Force -ErrorAction SilentlyContinue
            }
        }
    }
}

# ---------------------------------------------------------------- 模型

function MS-Url([string]$Repo, [string]$Path) {
    "https://modelscope.cn/api/v1/models/$Repo/repo?Revision=master&FilePath=$Path"
}

# 图像处理需要的四个文件(见 sd.cpp 的 docs/qwen_image_2.1.md)
$imageModels = @(
    @{ Name = 'qwen_image_2.1-Q4_K.gguf';            Dir = $dirModel; Url = (MS-Url 'leejet/Qwen-Image-2.1-GGUF' 'qwen_image_2.1-Q4_K.gguf');             What = '扩散主干 Q4_K(约 4.0 GB)' }
    @{ Name = 'Qwen3VL-8B-Instruct-Q4_K_M.gguf';     Dir = $dirModel; Url = (MS-Url 'Qwen/Qwen3-VL-8B-Instruct-GGUF' 'Qwen3VL-8B-Instruct-Q4_K_M.gguf'); What = 'text encoder Qwen3-VL-8B Q4_K_M(约 4.8 GB)' }
    @{ Name = 'qwen_image_2.1_vae_bf16.safetensors'; Dir = $dirModel; Url = (MS-Url 'Comfy-Org/Qwen-Image-2.1' 'vae/qwen_image_2.1_vae_bf16.safetensors'); What = 'VAE bf16(约 644 MB)' }
)
if (-not $SkipVision) {
    $imageModels += @{ Name = 'mmproj-Qwen3VL-8B-Instruct-F16.gguf'; Dir = $dirModel; Url = (MS-Url 'Qwen/Qwen3-VL-8B-Instruct-GGUF' 'mmproj-Qwen3VL-8B-Instruct-F16.gguf'); What = '视觉投影层 mmproj(约 1.1 GB,图片编辑用)' }
}

# 编程处理:Qwen3-Coder-30B-A3B(MoE,30B 总参 / 3B 激活)的 4-bit 量化版。
# 官方没有出 GGUF,ModelScope 上可用的是 unsloth 的转换。
$codeModels = @(
    @{ Name = 'Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf'; Dir = $dirModel; Url = (MS-Url 'unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF' 'Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf'); What = '编程模型 Qwen3-Coder-30B-A3B Q4_K_M(约 17.3 GB)' }
)

$models = @()
if (-not $CodeOnly) { $models += $imageModels }
if (-not $ImageOnly) { $models += $codeModels }

if (-not $BinariesOnly) {
    $totalMb = 0
    Write-Host ''
    Write-Host '=== 模型(来自 ModelScope)===' -ForegroundColor Cyan
    foreach ($m in $models) {
        Write-Host ("- {0}" -f $m.What)
        $dest = Join-Path $m.Dir $m.Name
        Fetch-File -Url $m.Url -Dest $dest -What $m.What | Out-Null
        if (Test-Path $dest) { $totalMb += (Get-Item $dest).Length / 1MB }
    }
    Write-Host ''
    Write-Host ("本次涉及 {0:N0} MB" -f $totalMb) -ForegroundColor DarkGray
}

# ---------------------------------------------------------------- 汇总

Write-Host ''
Write-Host '=== 结果 ===' -ForegroundColor Cyan
foreach ($d in $dirLlama, $dirSd, $dirModel, $dirFfmpeg) {
    $files = @(Get-ChildItem $d -File -ErrorAction SilentlyContinue)
    $sum = ($files | Measure-Object Length -Sum).Sum
    if (-not $sum) { $sum = 0 }
    Write-Host ("{0,-22} {1,3} 个文件  {2,9:N0} MB" -f (Split-Path $d -Leaf), $files.Count, ($sum / 1MB))
}
Write-Host ''
Write-Host 'llama.cpp 可执行文件:'
Get-ChildItem $dirLlama -Filter '*.exe' -ErrorAction SilentlyContinue | Select-Object -First 10 | ForEach-Object { "  $($_.Name)" }
Write-Host 'sd.cpp 可执行文件:'
Get-ChildItem $dirSd -Filter '*.exe' -ErrorAction SilentlyContinue | ForEach-Object { "  $($_.Name)" }