lxy/u-desk
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复: SFTP连接首拉不显示与传输挂起

Browse Source 
- watcher握手期伪触发致幻影本地列表，根因修复
- 连接探测2.5s超时竞速，握手10s预算
- 30s保活自动掐死半开连接，取连接时剔除已断开
- 副本命名与进度计数抽公共，下载缓存清理去重
This commit is contained in:
lxy
2026-09-15 22:26:22 +08:00
parent e7a2c5148e
commit 6ea9d9ac99
9 changed files with 731 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/ossdrv/service.go
+81 -49
View File
@@ -5,6 +5,7 @@ import (
"context"
"encoding/base64"
"fmt"
"log"
"os"
"path"
"path/filepath"
@@ -12,6 +13,7 @@ import (
"sync"
"time"
"u-desk/internal/common"
"u-desk/internal/filesystem"
"u-desk/internal/oss"
"u-desk/internal/oss/aliyun"
@@ -112,17 +114,12 @@ func (m *Manager) getOrCreateBucketClient(provider, bucket, region string) (oss.
UseHTTPS: true,
})
case "aliyun":
// 有桶级 region 时不传账户 Endpoint，让 NewClient 从 region 派生正确的 endpoint
ep := c.Endpoint
if region != "" {
ep = ""
}
client, err = aliyun.NewClient(&aliyun.Config{
AccessKeyID: c.AccessKey,
AccessKeySecret: c.SecretKey,
Bucket: bucket,
Region: region,
Endpoint: ep,
Endpoint: c.Endpoint,
UseHTTPS: true,
})
default:
@@ -367,14 +364,14 @@ func (s *Service) ReadFile(connID string, rawPath string) (string, error) {
return "", fmt.Errorf("获取文件信息失败: %w", err)
}
if info.Size > maxSize {
return "", fmt.Errorf("文件过大 (%s)，超过 %d 限制", filesystem.FormatBytes(info.Size), maxSize)
return "", fmt.Errorf("文件过大 (%s)，超过 %d 限制", common.FormatBytesInt(info.Size), maxSize)
}
var buf bytes.Buffer
if err := c.Download(ctx, key, &buf); err != nil {
return "", fmt.Errorf("读取文件失败: %w", err)
}
return filesystem.BytesToString(buf.Bytes()), nil
return common.BytesToString(buf.Bytes()), nil
}
// WriteFile 写入文件内容
@@ -442,7 +439,7 @@ func (s *Service) GetFileInfo(connID string, rawPath string) (map[string]interfa
"name": path.Base(info.Key),
"path": bucketPrefix + info.Key,
"size": info.Size,
"size_str": filesystem.FormatBytes(info.Size),
"size_str": common.FormatBytesInt(info.Size),
"is_dir": strings.HasSuffix(info.Key, "/"),
"mod_time": info.LastModified.Format("2006-01-02 15:04:05"),
}, nil
@@ -474,7 +471,7 @@ func (s *Service) CreateDir(connID string, rawPath string) (*filesystem.FileOper
Path: "/" + bucket + "/" + key,
Name: name,
IsDir: true,
SizeStr: filesystem.FormatBytes(0),
SizeStr: common.FormatBytesInt(0),
}, nil
}
@@ -499,7 +496,7 @@ func (s *Service) CreateFile(connID string, rawPath string) (*filesystem.FileOpe
Path: "/" + bucket + "/" + key,
Name: path.Base(key),
IsDir: false,
SizeStr: filesystem.FormatBytes(0),
SizeStr: common.FormatBytesInt(0),
}, nil
}
@@ -661,22 +658,36 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
ctx := context.Background()
// 1. 创建临时目录，保留 OSS 目录结构
tmpDir, err := os.MkdirTemp("", "udesk-site-*")
if err != nil {
return "", fmt.Errorf("创建临时目录失败: %w", err)
// 1. 缓存检查：用确定性路径，命中直接返回
htmlInfo, _ := c.GetFileInfo(ctx, key)
var htmlSize int64
var htmlModTime string
if htmlInfo != nil {
htmlSize = htmlInfo.Size
htmlModTime = htmlInfo.LastModified.Format("2006-01-02 15:04:05")
}
cacheDir := common.SiteCacheDir("oss", connID, rawPath, htmlSize, htmlModTime)
keyDir := path.Dir(key)
var htmlLocalPath string
var htmlCachePath string
if keyDir != "" && keyDir != "." {
htmlLocalPath = filepath.Join(tmpDir, filepath.FromSlash(keyDir), path.Base(key))
if err := os.MkdirAll(filepath.Dir(htmlLocalPath), 0755); err != nil {
os.RemoveAll(tmpDir)
return "", fmt.Errorf("创建目录失败: %w", err)
}
htmlCachePath = filepath.Join(cacheDir, filepath.FromSlash(keyDir), path.Base(key))
} else {
htmlLocalPath = filepath.Join(tmpDir, path.Base(key))
htmlCachePath = filepath.Join(cacheDir, path.Base(key))
}
if _, err := os.Stat(htmlCachePath); err == nil {
return htmlCachePath, nil // 缓存命中
}
os.RemoveAll(cacheDir)
if err := os.MkdirAll(cacheDir, 0755); err != nil {
return "", fmt.Errorf("创建缓存目录失败: %w", err)
}
tmpDir := cacheDir
htmlLocalPath := htmlCachePath
if err := os.MkdirAll(filepath.Dir(htmlLocalPath), 0755); err != nil {
os.RemoveAll(tmpDir)
return "", fmt.Errorf("创建目录失败: %w", err)
}
// 2. 下载 HTML
@@ -697,7 +708,7 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
if err != nil {
return htmlLocalPath, nil // HTML 已下载，资源解析失败不影响
}
resources := filesystem.ExtractHtmlResources(string(htmlContent))
resources := common.ExtractHtmlResources(string(htmlContent))
// 4. 下载资源
htmlOssDir := keyDir
@@ -717,8 +728,11 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
}
}
// 绝对路径资源：串行嗅探 siteRoot，相对路径：并行下载
type relativeTask struct{ ossKey, localPath string }
var relativeTasks []relativeTask
for _, resPath := range resources {
if filesystem.ShouldSkipResource(resPath) {
if common.ShouldSkipResource(resPath) {
continue
}
@@ -743,11 +757,19 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
} else {
ossKey = cleanPath
}
if downloadResource(c, ctx, ossKey, localPath) {
recordDir(ossKey)
}
relativeTasks = append(relativeTasks, relativeTask{ossKey, localPath})
}
}
// 相对路径资源并行下载（recordDir 需加锁，多 goroutine 并发回调）
var mu sync.Mutex
common.RunConcurrent(relativeTasks, common.SiteDownloadConcurrency, func(t relativeTask) {
if downloadResource(c, ctx, t.ossKey, t.localPath) {
mu.Lock()
recordDir(t.ossKey)
mu.Unlock()
}
})
// 5. 补充下载已发现目录中的剩余文件（覆盖 webpack 动态 chunk 等）
for _, dir := range discoveredDirs {
supplementDir(c, ctx, dir, tmpDir, siteRoot)
@@ -756,7 +778,6 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
return htmlLocalPath, nil
}
// resolveAbsoluteResourcePath 解析绝对路径资源，首次嗅探网站根，后续直接使用
// resolveAndDownload 解析绝对路径并下载：首次嗅探网站根，后续直接使用
func resolveAndDownload(c oss.OSSProvider, ctx context.Context, htmlOssDir string, cleanPath string, localPath string, siteRoot *string) string {
if *siteRoot != "" {
@@ -815,37 +836,48 @@ func downloadResource(c oss.OSSProvider, ctx context.Context, ossKey string, loc
}
// supplementDir 补充下载远程目录中尚未下载的文件（只处理已知资源所在目录）
// 数量与单文件大小上限见 common.MaxSupplementFiles / common.MaxSupplementFileSize
func supplementDir(c oss.OSSProvider, ctx context.Context, remoteDir string, tmpDir string, siteRoot string) {
prefix := remoteDir + "/"
result, err := c.ListFiles(ctx, &oss.ListOptions{Prefix: prefix, MaxKeys: 200})
if err != nil {
var marker string
count := 0
for count < common.MaxSupplementFiles {
result, err := c.ListFiles(ctx, &oss.ListOptions{Prefix: prefix, MaxKeys: 200, Marker: marker})
if err != nil {
return
}
for _, f := range result.Files {
if strings.HasSuffix(f.Key, "/") || f.Size == 0 {
continue
}
localPath := filepath.Join(tmpDir, filepath.FromSlash(f.Key))
if _, err := os.Stat(localPath); err == nil {
continue
for _, f := range result.Files {
if strings.HasSuffix(f.Key, "/") || f.Size == 0 || f.Size > common.MaxSupplementFileSize {
continue
}
if count >= common.MaxSupplementFiles {
log.Printf("[站点下载] 补充扫描达到上限: count=%d", common.MaxSupplementFiles)
return
}
localPath := filepath.Join(tmpDir, filepath.FromSlash(f.Key))
if _, err := os.Stat(localPath); err == nil {
continue
}
if downloadResource(c, ctx, f.Key, localPath) {
count++
}
}
downloadResource(c, ctx, f.Key, localPath)
if !result.IsTruncated {
return
}
marker = result.NextMarker
}
}
// DownloadToTemp 下载文件到本地临时目录（带 SQLite 缓存）
func (s *Service) DownloadToTemp(connID string, rawPath string) (string, error) {
// 先获取文件元信息用于缓存键，确保远程文件变更时能淘汰旧缓存
var fileSize int64
var modTime string
if info, err := s.GetFileInfo(connID, rawPath); err == nil {
if sz, ok := info["size"].(int64); ok {
fileSize = sz
}
if mt, ok := info["mod_time"].(string); ok {
modTime = mt
}
// 获取文件元信息用于缓存键
info, err := s.GetFileInfo(connID, rawPath)
if err != nil {
return "", fmt.Errorf("获取文件信息失败: %w", err)
}
fileSize, _ := info["size"].(int64)
modTime, _ := info["mod_time"].(string)
return storage.DownloadToTempCached("oss", connID, rawPath, fileSize, modTime, func() (string, error) {
return s.downloadToTempDirect(connID, rawPath)
})
@@ -920,7 +952,7 @@ func toOssOperationResult(m map[string]interface{}, isDir bool) *filesystem.File
Path: p,
Name: name,
Size: size,
SizeStr: filesystem.FormatBytes(size),
SizeStr: common.FormatBytesInt(size),
IsDir: isDir,
ModTime: modTime,
}