修复: .GC 不应入库(曾覆盖远程仓库)+ 模型读取优化
This commit is contained in:
1 parent
f69a8fbb7f
commit
936aec8185
279 files changed
+182
-9991
No files matched your search
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
@@ -118,15 +119,84 @@ public sealed class GitCodeRemote : IModelRemote
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>递归列出目录树(含目录项与文件项)。</summary>
|
||||
/// <summary>
|
||||
/// 递归列出目录树(含目录项与文件项)。
|
||||
/// 每个子目录一次 HTTP 请求,因此并行展开同一层的所有目录,
|
||||
/// 否则 3 个模型、十几个目录串行请求会非常慢。
|
||||
/// </summary>
|
||||
public async Task<IReadOnlyList<RepoEntry>> ListTreeAsync(string rootPath, CancellationToken ct)
|
||||
{
|
||||
var result = new List<RepoEntry>();
|
||||
var bag = new System.Collections.Concurrent.ConcurrentBag<RepoEntry>();
|
||||
var repoPath = (rootPath ?? _modelDir).Trim('/');
|
||||
await WalkAsync(repoPath, "", 0, result, ct);
|
||||
await WalkParallelAsync(repoPath, "", 0, bag, ct);
|
||||
|
||||
// 稳定排序:目录在前、按路径字典序,保证调用方拿到确定顺序
|
||||
result = bag.OrderBy(e => e.IsDirectory ? 0 : 1)
|
||||
.ThenBy(e => e.RelativePath, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task WalkParallelAsync(string repoPath, string relative, int depth,
|
||||
System.Collections.Concurrent.ConcurrentBag<RepoEntry> bag, CancellationToken ct)
|
||||
{
|
||||
if (depth > 8) return;
|
||||
var url = _api + repoPath + "?ref=" + _branch;
|
||||
string text;
|
||||
try { text = await GetWithRetryAsync(url, ct); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"列目录失败 {repoPath}: {ex.Message}");
|
||||
return;
|
||||
}
|
||||
if (!text.TrimStart().StartsWith("[", StringComparison.Ordinal)) return;
|
||||
|
||||
var subDirs = new List<(string Path, string Relative)>();
|
||||
|
||||
using (var doc = JsonDocument.Parse(text))
|
||||
{
|
||||
foreach (var item in doc.RootElement.EnumerateArray())
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var name = Str(item, "name");
|
||||
if (string.IsNullOrEmpty(name) || name.StartsWith('.')) continue;
|
||||
// 这些目录不是模型运行数据,跳过可显著减少请求
|
||||
// (模型目录里常见的 src/ bin/ obj/ 是源码与构建产物)
|
||||
if (IsIgnoredDir(name)) continue;
|
||||
var path = Str(item, "path");
|
||||
if (string.IsNullOrWhiteSpace(path)) path = repoPath + "/" + name;
|
||||
var isDir = Str(item, "type") == "dir";
|
||||
var childRelative = relative.Length == 0 ? name : relative + "/" + name;
|
||||
|
||||
bag.Add(new RepoEntry
|
||||
{
|
||||
Name = name,
|
||||
Path = path!,
|
||||
RelativePath = childRelative,
|
||||
IsDirectory = isDir,
|
||||
DownloadUrl = NullableStr(item, "download_url"),
|
||||
Sha = NullableStr(item, "sha"),
|
||||
Size = Num(item, "size")
|
||||
});
|
||||
|
||||
if (isDir) subDirs.Add((path!, childRelative));
|
||||
}
|
||||
}
|
||||
|
||||
if (subDirs.Count == 0) return;
|
||||
|
||||
// 同层并行;并发保持很低——GitCode API 对突发请求会返回 429
|
||||
using var gate = new SemaphoreSlim(2);
|
||||
var tasks = subDirs.Select(async d =>
|
||||
{
|
||||
await gate.WaitAsync(ct);
|
||||
try { await WalkParallelAsync(d.Path, d.Relative, depth + 1, bag, ct); }
|
||||
finally { gate.Release(); }
|
||||
});
|
||||
await Task.WhenAll(tasks);
|
||||
}
|
||||
|
||||
private async Task WalkAsync(string repoPath, string relative, int depth,
|
||||
List<RepoEntry> result, CancellationToken ct)
|
||||
{
|
||||
@@ -161,6 +231,43 @@ public sealed class GitCodeRemote : IModelRemote
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>遍历时忽略的目录:源码与构建产物不属于模型运行数据。</summary>
|
||||
private static bool IsIgnoredDir(string name) => name switch
|
||||
{
|
||||
"src" => true,
|
||||
"bin" => true,
|
||||
"obj" => true,
|
||||
"node_modules" => true,
|
||||
".git" => true,
|
||||
".vs" => true,
|
||||
_ => false
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// 带退避重试的 GET。GitCode API 对突发请求返回 429,
|
||||
/// 这里做 3 次指数退避,避免并行遍历时直接失败。
|
||||
/// </summary>
|
||||
private async Task<string> GetWithRetryAsync(string url, CancellationToken ct,
|
||||
int maxAttempts = 3)
|
||||
{
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await _http.GetStringAsync(url, ct);
|
||||
}
|
||||
catch (HttpRequestException ex) when (attempt < maxAttempts)
|
||||
{
|
||||
// 429 / 5xx 才重试,其余直接抛
|
||||
var retriable = ex.StatusCode is null ||
|
||||
(int)ex.StatusCode == 429 ||
|
||||
(int)ex.StatusCode >= 500;
|
||||
if (!retriable) throw;
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(400 * attempt * attempt), ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<byte[]> DownloadAsync(RepoEntry entry, CancellationToken ct)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(entry.DownloadUrl))
|
||||
|
||||
@@ -217,30 +217,43 @@ public sealed class UpdateService
|
||||
try
|
||||
{
|
||||
var total = files.Sum(f => Math.Max(1L, f.Size));
|
||||
// 并行下载:模型包通常有几十个文件,串行下载会明显拖慢启动。
|
||||
// 并发上限 6,避免被服务端限流;进度用原子计数汇总。
|
||||
long done = 0;
|
||||
foreach (var entry in files)
|
||||
var stagingFull = Path.GetFullPath(staging)
|
||||
.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar;
|
||||
using var gate = new SemaphoreSlim(6);
|
||||
|
||||
var tasks = files.Select(async entry =>
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var bytes = await _remote.DownloadAsync(entry, ct);
|
||||
// 路径穿越防护:规范化后必须仍落在 staging 目录内,
|
||||
// 防止远端清单里的 "../" 或绝对路径写进安装目录之外。
|
||||
var dest = Path.GetFullPath(Path.Combine(staging,
|
||||
entry.RelativePath.Replace('/', Path.DirectorySeparatorChar)));
|
||||
var stagingFull = Path.GetFullPath(staging)
|
||||
.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar;
|
||||
if (!dest.StartsWith(stagingFull, StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException($"非法的模型文件路径: {entry.RelativePath}");
|
||||
var destDir = Path.GetDirectoryName(dest);
|
||||
if (!string.IsNullOrEmpty(destDir)) Directory.CreateDirectory(destDir);
|
||||
await File.WriteAllBytesAsync(dest, bytes, ct);
|
||||
done += bytes.Length;
|
||||
progress?.Report(new UpdateProgress
|
||||
await gate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
Value = 0.05 + 0.72 * (index + Math.Min(1, (double)done / Math.Max(1, total))) / Math.Max(1, count),
|
||||
Message = L.I.F("update.downloadingModel", model.Id),
|
||||
Detail = entry.RelativePath
|
||||
});
|
||||
}
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var bytes = await _remote.DownloadAsync(entry, ct);
|
||||
|
||||
// 路径穿越防护
|
||||
var dest = Path.GetFullPath(Path.Combine(staging,
|
||||
entry.RelativePath.Replace('/', Path.DirectorySeparatorChar)));
|
||||
if (!dest.StartsWith(stagingFull, StringComparison.OrdinalIgnoreCase))
|
||||
throw new InvalidOperationException($"非法的模型文件路径: {entry.RelativePath}");
|
||||
|
||||
var destDir = Path.GetDirectoryName(dest);
|
||||
if (!string.IsNullOrEmpty(destDir)) Directory.CreateDirectory(destDir);
|
||||
await File.WriteAllBytesAsync(dest, bytes, ct);
|
||||
|
||||
var now = Interlocked.Add(ref done, bytes.Length);
|
||||
progress?.Report(new UpdateProgress
|
||||
{
|
||||
Value = 0.05 + 0.72 * (index + Math.Min(1, (double)now / Math.Max(1, total))) / Math.Max(1, count),
|
||||
Message = L.I.F("update.downloadingModel", model.Id),
|
||||
Detail = entry.RelativePath
|
||||
});
|
||||
}
|
||||
finally { gate.Release(); }
|
||||
});
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
|
||||
// 原子替换:先备份旧目录,成功后再删备份;失败自动回滚
|
||||
Directory.CreateDirectory(_paths.ModelRoot);
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -13,7 +13,7 @@ using System.Reflection;
|
||||
[assembly: System.Reflection.AssemblyCompanyAttribute("NeoRF")]
|
||||
[assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")]
|
||||
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.4.0")]
|
||||
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.4+5e0b46b3f3cd9800af177571c7809a7cdc8e9249")]
|
||||
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.4+f69a8fbb7fa563800802d3c2e319ed56365c3b8c")]
|
||||
[assembly: System.Reflection.AssemblyProductAttribute("Earth Model Studio")]
|
||||
[assembly: System.Reflection.AssemblyTitleAttribute("EarthModelStudio")]
|
||||
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.4.0")]
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
4b53b80918cdf3129cb1e2a40db5bf8bd2373753f7b100d149cfe4108f55dfbd
|
||||
7915cc6a645d81627b594db52c0bcb43aed3975e57c9edf46061b1902e2a7193
|
||||
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+1
-1
@@ -13,7 +13,7 @@ using System.Reflection;
|
||||
[assembly: System.Reflection.AssemblyCompanyAttribute("NeoRF")]
|
||||
[assembly: System.Reflection.AssemblyConfigurationAttribute("Release")]
|
||||
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.4.0")]
|
||||
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.4+5e0b46b3f3cd9800af177571c7809a7cdc8e9249")]
|
||||
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.4+f69a8fbb7fa563800802d3c2e319ed56365c3b8c")]
|
||||
[assembly: System.Reflection.AssemblyProductAttribute("Earth Model Studio")]
|
||||
[assembly: System.Reflection.AssemblyTitleAttribute("EarthModelStudio")]
|
||||
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.4.0")]
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
bbea3a678295bbbef711b3c94168d9cc4ab226467bb6869e4e9f13afae159277
|
||||
f5f6f7aaca707ee47caaff226938ba40218288fdd12005017dae3507e893fbe2
|
||||
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+1
-1
@@ -13,7 +13,7 @@ using System.Reflection;
|
||||
[assembly: System.Reflection.AssemblyCompanyAttribute("NeoRF")]
|
||||
[assembly: System.Reflection.AssemblyConfigurationAttribute("Release")]
|
||||
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.4.0")]
|
||||
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.4+5e0b46b3f3cd9800af177571c7809a7cdc8e9249")]
|
||||
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.4+f69a8fbb7fa563800802d3c2e319ed56365c3b8c")]
|
||||
[assembly: System.Reflection.AssemblyProductAttribute("Earth Model Studio")]
|
||||
[assembly: System.Reflection.AssemblyTitleAttribute("EarthModelStudio")]
|
||||
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.4.0")]
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
bbea3a678295bbbef711b3c94168d9cc4ab226467bb6869e4e9f13afae159277
|
||||
f5f6f7aaca707ee47caaff226938ba40218288fdd12005017dae3507e893fbe2
|
||||
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
+1
-1
@@ -13,7 +13,7 @@ using System.Reflection;
|
||||
[assembly: System.Reflection.AssemblyCompanyAttribute("NeoRF")]
|
||||
[assembly: System.Reflection.AssemblyConfigurationAttribute("Release")]
|
||||
[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.4.0")]
|
||||
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.4+5e0b46b3f3cd9800af177571c7809a7cdc8e9249")]
|
||||
[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.4+f69a8fbb7fa563800802d3c2e319ed56365c3b8c")]
|
||||
[assembly: System.Reflection.AssemblyProductAttribute("Earth Model Studio")]
|
||||
[assembly: System.Reflection.AssemblyTitleAttribute("EarthModelStudio")]
|
||||
[assembly: System.Reflection.AssemblyVersionAttribute("1.0.4.0")]
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
bbea3a678295bbbef711b3c94168d9cc4ab226467bb6869e4e9f13afae159277
|
||||
f5f6f7aaca707ee47caaff226938ba40218288fdd12005017dae3507e893fbe2
|
||||
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Reference in new issue
Block a user