修复: SFTP连接首拉不显示与传输挂起
- watcher握手期伪触发致幻影本地列表,根因修复 - 连接探测2.5s超时竞速,握手10s预算 - 30s保活自动掐死半开连接,取连接时剔除已断开 - 副本命名与进度计数抽公共,下载缓存清理去重
This commit is contained in:
+18
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -124,6 +126,22 @@ func (a *App) SftpDownloadToTempCached(connID string, remotePath string, fileSiz
|
||||
return a.ensureSftpService().DownloadToTempCached(connID, remotePath, fileSize, modTime)
|
||||
}
|
||||
|
||||
// SftpDownloadToFile 下载远程文件/目录到用户指定目录(流式无上限,重名自动副本),返回本地落地路径
|
||||
func (a *App) SftpDownloadToFile(connID string, remotePath string, localDstDir string) (string, error) {
|
||||
emitter := newTransferEmitter(a, "download", path.Base(remotePath))
|
||||
localPath, err := a.ensureSftpService().DownloadToFile(connID, remotePath, localDstDir, emitter.progress)
|
||||
emitter.finish(err)
|
||||
return localPath, err
|
||||
}
|
||||
|
||||
// SftpUploadFromFile 上传本地文件/目录到远程目录(流式,目录递归,重名自动副本)
|
||||
func (a *App) SftpUploadFromFile(connID string, localPath string, remoteDir string) error {
|
||||
emitter := newTransferEmitter(a, "upload", filepath.Base(localPath))
|
||||
err := a.ensureSftpService().UploadFromFile(connID, localPath, remoteDir, emitter.progress)
|
||||
emitter.finish(err)
|
||||
return err
|
||||
}
|
||||
|
||||
// SftpGetCommonPaths 获取 SFTP 远程主机常用路径
|
||||
func (a *App) SftpGetCommonPaths(connID string) (map[string]string, error) {
|
||||
return a.ensureSftpService().GetCommonPaths(connID)
|
||||
|
||||
+189
-2
@@ -1,8 +1,20 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
// InterfaceSliceToStringSlice 将 []interface{} 安全转换为 []string
|
||||
@@ -16,8 +28,7 @@ func InterfaceSliceToStringSlice(slice []interface{}) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
// FormatBytes 格式化字节大小为人类可读格式
|
||||
// 例如: 1024 → "1.00 KB", 1048576 → "1.00 MB"
|
||||
// FormatBytes 格式化字节大小为人类可读格式(uint64 版本)
|
||||
func FormatBytes(bytes uint64) string {
|
||||
const unit = 1024
|
||||
if bytes < unit {
|
||||
@@ -31,6 +42,104 @@ func FormatBytes(bytes uint64) string {
|
||||
return fmt.Sprintf("%.2f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
||||
}
|
||||
|
||||
// FormatBytesInt 格式化字节大小(int64 版本,用于文件大小)
|
||||
func FormatBytesInt(bytes int64) string {
|
||||
return FormatBytes(uint64(bytes))
|
||||
}
|
||||
|
||||
// CopyFile 复制文件内容
|
||||
func CopyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
_, err = out.ReadFrom(in)
|
||||
return err
|
||||
}
|
||||
|
||||
// BytesToString 智能编码转换:UTF-8 直接返回,否则尝试 GBK → UTF-8
|
||||
func BytesToString(data []byte) string {
|
||||
if utf8.Valid(data) {
|
||||
return string(data)
|
||||
}
|
||||
reader := transform.NewReader(bytes.NewReader(data), simplifiedchinese.GBK.NewDecoder())
|
||||
decoded, err := io.ReadAll(reader)
|
||||
if err != nil || !utf8.Valid(decoded) {
|
||||
return string(data)
|
||||
}
|
||||
return string(decoded)
|
||||
}
|
||||
|
||||
var (
|
||||
htmlResRegex = regexp.MustCompile(`(?:src|href|data-src|data-url|poster)=["']([^"']+)["']`)
|
||||
htmlSrcsetRegex = regexp.MustCompile(`srcset\s*=\s*["']([^"']+)["']`)
|
||||
htmlCssUrlRe = regexp.MustCompile(`url\(\s*["']?([^"')]+)["']?\s*\)`)
|
||||
)
|
||||
|
||||
// parseSrcsetUrls 从 srcset 属性值中提取各 URL
|
||||
// srcset 格式: "small.jpg 480w, large.jpg 1024w" → ["small.jpg", "large.jpg"]
|
||||
func parseSrcsetUrls(srcset string) []string {
|
||||
var urls []string
|
||||
for _, candidate := range strings.Split(srcset, ",") {
|
||||
parts := strings.Fields(strings.TrimSpace(candidate))
|
||||
if len(parts) > 0 {
|
||||
urls = append(urls, parts[0])
|
||||
}
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
// ExtractHtmlResources 从 HTML 内容提取资源路径
|
||||
func ExtractHtmlResources(html string) []string {
|
||||
seen := make(map[string]bool)
|
||||
var resources []string
|
||||
add := func(v string) {
|
||||
v = strings.TrimSpace(v)
|
||||
if v != "" && !seen[v] {
|
||||
seen[v] = true
|
||||
resources = append(resources, v)
|
||||
}
|
||||
}
|
||||
for _, m := range htmlResRegex.FindAllStringSubmatch(html, -1) {
|
||||
if len(m) > 1 {
|
||||
add(m[1])
|
||||
}
|
||||
}
|
||||
for _, m := range htmlSrcsetRegex.FindAllStringSubmatch(html, -1) {
|
||||
if len(m) > 1 {
|
||||
for _, u := range parseSrcsetUrls(m[1]) {
|
||||
add(u)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range htmlCssUrlRe.FindAllStringSubmatch(html, -1) {
|
||||
if len(m) > 1 {
|
||||
add(m[1])
|
||||
}
|
||||
}
|
||||
return resources
|
||||
}
|
||||
|
||||
// ShouldSkipResource 判断资源路径是否应跳过
|
||||
func ShouldSkipResource(p string) bool {
|
||||
return strings.HasPrefix(p, "data:") ||
|
||||
strings.HasPrefix(p, "http://") ||
|
||||
strings.HasPrefix(p, "https://") ||
|
||||
strings.HasPrefix(p, "//") ||
|
||||
strings.HasPrefix(p, "#") ||
|
||||
strings.HasPrefix(p, "javascript:") ||
|
||||
strings.HasPrefix(p, "mailto:") ||
|
||||
strings.HasPrefix(p, "blob:")
|
||||
}
|
||||
|
||||
// Contains 检查切片是否包含元素
|
||||
func Contains[T comparable](slice []T, item T) bool {
|
||||
for _, s := range slice {
|
||||
@@ -61,3 +170,81 @@ func IsWindows() bool {
|
||||
return runtime.GOOS == "windows"
|
||||
}
|
||||
|
||||
// siteCacheDir 返回站点预览的确定性缓存目录路径
|
||||
func SiteCacheDir(transport, connID, remotePath string, fileSize int64, modTime string) string {
|
||||
h := sha256.New()
|
||||
h.Write([]byte(fmt.Sprintf("site:%s:%s:%s:%d:%s", transport, connID, remotePath, fileSize, modTime)))
|
||||
hash := fmt.Sprintf("%x", h.Sum(nil))[:16]
|
||||
return filepath.Join(os.TempDir(), "u-desk-cache", "site-"+hash)
|
||||
}
|
||||
|
||||
// 站点预览补充下载限制(sftp 与 ossdrv 共用)
|
||||
const (
|
||||
MaxSupplementFiles = 50 // 每目录补充下载文件数上限
|
||||
MaxSupplementFileSize = 5 << 20 // 单文件补充下载大小上限(5MB)
|
||||
SiteDownloadConcurrency = 8 // 相对路径资源并行下载并发数
|
||||
)
|
||||
|
||||
// RunConcurrent 以最多 limit 个并发执行 tasks,全部完成后返回
|
||||
// fn 在独立 goroutine 中运行,需自行处理错误与共享状态的加锁
|
||||
func RunConcurrent[T any](tasks []T, limit int, fn func(t T)) {
|
||||
if len(tasks) == 0 || limit <= 0 {
|
||||
return
|
||||
}
|
||||
if limit > len(tasks) {
|
||||
limit = len(tasks)
|
||||
}
|
||||
sem := make(chan struct{}, limit)
|
||||
var wg sync.WaitGroup
|
||||
for _, task := range tasks {
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(t T) {
|
||||
defer func() { <-sem; wg.Done() }()
|
||||
fn(t)
|
||||
}(task)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// UniqueName 返回不与现有条目冲突的文件名:重名时追加「 - 副本」「 - 副本 (2)」后缀(扩展名前插入)。
|
||||
// exists 报告候选文件名是否已被占用(本地文件系统与 SFTP 各自提供实现)。
|
||||
func UniqueName(name string, exists func(candidate string) (bool, error)) (string, error) {
|
||||
taken, err := exists(name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !taken {
|
||||
return name, nil
|
||||
}
|
||||
ext := filepath.Ext(name)
|
||||
base := strings.TrimSuffix(name, ext)
|
||||
for i := 1; ; i++ {
|
||||
suffix := " - 副本"
|
||||
if i > 1 {
|
||||
suffix = fmt.Sprintf(" - 副本 (%d)", i)
|
||||
}
|
||||
candidate := base + suffix + ext
|
||||
taken, err := exists(candidate)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !taken {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CountingReader 包装读取器,按实际读取字节数回调 OnN(n > 0 时)
|
||||
type CountingReader struct {
|
||||
R io.Reader
|
||||
OnN func(n int64)
|
||||
}
|
||||
|
||||
func (cr *CountingReader) Read(buf []byte) (int, error) {
|
||||
n, err := cr.R.Read(buf)
|
||||
if n > 0 {
|
||||
cr.OnN(int64(n))
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -122,14 +123,7 @@ func (c *Client) canonicalizeOSSHeaders(headers map[string]string) string {
|
||||
}
|
||||
|
||||
// 按字典序排序
|
||||
// 这里简单处理,实际应该用排序算法
|
||||
for i := 0; i < len(ossHeaders); i++ {
|
||||
for j := i + 1; j < len(ossHeaders); j++ {
|
||||
if ossHeaders[i] > ossHeaders[j] {
|
||||
ossHeaders[i], ossHeaders[j] = ossHeaders[j], ossHeaders[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(ossHeaders)
|
||||
|
||||
// 拼接结果
|
||||
if len(ossHeaders) == 0 {
|
||||
@@ -315,13 +309,14 @@ func (c *Client) GetFileInfo(ctx context.Context, key string) (*oss.FileInfo, er
|
||||
size := resp.ContentLength
|
||||
etag := resp.Header.Get("ETag")
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
_ = resp.Header.Get("Last-Modified") // 预留,可能需要解析时间
|
||||
lastModified, _ := http.ParseTime(resp.Header.Get("Last-Modified"))
|
||||
|
||||
return &oss.FileInfo{
|
||||
Key: key,
|
||||
Size: size,
|
||||
ETag: strings.Trim(etag, "\""),
|
||||
ContentType: contentType,
|
||||
LastModified: lastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -487,26 +482,23 @@ func (c *Client) DeleteMultiple(ctx context.Context, keys []string) (*oss.Delete
|
||||
|
||||
// GetSignedURL 获取预签名URL
|
||||
func (c *Client) GetSignedURL(ctx context.Context, key string, expiresIn time.Duration) (string, error) {
|
||||
// 阿里云 OSS 使用签名 URL
|
||||
// 格式: ?OSSAccessKeyId=xxx&Expires=xxx&Signature=xxx
|
||||
expiration := time.Now().Add(expiresIn).Unix()
|
||||
|
||||
// 构建签名
|
||||
// 签名计算用原始 key(不编码)
|
||||
path := "/" + c.config.Bucket + "/" + key
|
||||
stringToSign := "GET\n\n\n" + fmt.Sprintf("%d", expiration) + "\n" + path
|
||||
h := hmac.New(sha1.New, []byte(c.config.AccessKeySecret))
|
||||
h.Write([]byte(stringToSign))
|
||||
signature := base64.StdEncoding.EncodeToString(h.Sum(nil))
|
||||
signature = strings.TrimRight(signature, "=") // URL Safe
|
||||
signature := base64.URLEncoding.EncodeToString(h.Sum(nil))
|
||||
|
||||
scheme := "https://"
|
||||
if !c.config.UseHTTPS {
|
||||
scheme = "http://"
|
||||
}
|
||||
baseURL := scheme + c.config.Bucket + "." + c.config.Endpoint + "/" + key
|
||||
baseURL := scheme + c.config.Bucket + "." + c.config.Endpoint + "/" + url.PathEscape(key)
|
||||
|
||||
signedURL := fmt.Sprintf("%s?OSSAccessKeyId=%s&Expires=%d&Signature=%s",
|
||||
baseURL, c.config.AccessKeyID, expiration, signature)
|
||||
baseURL, url.QueryEscape(c.config.AccessKeyID), expiration, url.QueryEscape(signature))
|
||||
|
||||
return signedURL, nil
|
||||
}
|
||||
|
||||
+71
-39
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"u-desk/internal/common"
|
||||
"u-desk/internal/filesystem"
|
||||
"u-desk/internal/oss"
|
||||
"u-desk/internal/oss/aliyun"
|
||||
@@ -112,17 +114,12 @@ func (m *Manager) getOrCreateBucketClient(provider, bucket, region string) (oss.
|
||||
UseHTTPS: true,
|
||||
})
|
||||
case "aliyun":
|
||||
// 有桶级 region 时不传账户 Endpoint,让 NewClient 从 region 派生正确的 endpoint
|
||||
ep := c.Endpoint
|
||||
if region != "" {
|
||||
ep = ""
|
||||
}
|
||||
client, err = aliyun.NewClient(&aliyun.Config{
|
||||
AccessKeyID: c.AccessKey,
|
||||
AccessKeySecret: c.SecretKey,
|
||||
Bucket: bucket,
|
||||
Region: region,
|
||||
Endpoint: ep,
|
||||
Endpoint: c.Endpoint,
|
||||
UseHTTPS: true,
|
||||
})
|
||||
default:
|
||||
@@ -367,14 +364,14 @@ func (s *Service) ReadFile(connID string, rawPath string) (string, error) {
|
||||
return "", fmt.Errorf("获取文件信息失败: %w", err)
|
||||
}
|
||||
if info.Size > maxSize {
|
||||
return "", fmt.Errorf("文件过大 (%s),超过 %d 限制", filesystem.FormatBytes(info.Size), maxSize)
|
||||
return "", fmt.Errorf("文件过大 (%s),超过 %d 限制", common.FormatBytesInt(info.Size), maxSize)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := c.Download(ctx, key, &buf); err != nil {
|
||||
return "", fmt.Errorf("读取文件失败: %w", err)
|
||||
}
|
||||
return filesystem.BytesToString(buf.Bytes()), nil
|
||||
return common.BytesToString(buf.Bytes()), nil
|
||||
}
|
||||
|
||||
// WriteFile 写入文件内容
|
||||
@@ -442,7 +439,7 @@ func (s *Service) GetFileInfo(connID string, rawPath string) (map[string]interfa
|
||||
"name": path.Base(info.Key),
|
||||
"path": bucketPrefix + info.Key,
|
||||
"size": info.Size,
|
||||
"size_str": filesystem.FormatBytes(info.Size),
|
||||
"size_str": common.FormatBytesInt(info.Size),
|
||||
"is_dir": strings.HasSuffix(info.Key, "/"),
|
||||
"mod_time": info.LastModified.Format("2006-01-02 15:04:05"),
|
||||
}, nil
|
||||
@@ -474,7 +471,7 @@ func (s *Service) CreateDir(connID string, rawPath string) (*filesystem.FileOper
|
||||
Path: "/" + bucket + "/" + key,
|
||||
Name: name,
|
||||
IsDir: true,
|
||||
SizeStr: filesystem.FormatBytes(0),
|
||||
SizeStr: common.FormatBytesInt(0),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -499,7 +496,7 @@ func (s *Service) CreateFile(connID string, rawPath string) (*filesystem.FileOpe
|
||||
Path: "/" + bucket + "/" + key,
|
||||
Name: path.Base(key),
|
||||
IsDir: false,
|
||||
SizeStr: filesystem.FormatBytes(0),
|
||||
SizeStr: common.FormatBytesInt(0),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -661,23 +658,37 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 1. 创建临时目录,保留 OSS 目录结构
|
||||
tmpDir, err := os.MkdirTemp("", "udesk-site-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("创建临时目录失败: %w", err)
|
||||
// 1. 缓存检查:用确定性路径,命中直接返回
|
||||
htmlInfo, _ := c.GetFileInfo(ctx, key)
|
||||
var htmlSize int64
|
||||
var htmlModTime string
|
||||
if htmlInfo != nil {
|
||||
htmlSize = htmlInfo.Size
|
||||
htmlModTime = htmlInfo.LastModified.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
cacheDir := common.SiteCacheDir("oss", connID, rawPath, htmlSize, htmlModTime)
|
||||
|
||||
keyDir := path.Dir(key)
|
||||
var htmlLocalPath string
|
||||
var htmlCachePath string
|
||||
if keyDir != "" && keyDir != "." {
|
||||
htmlLocalPath = filepath.Join(tmpDir, filepath.FromSlash(keyDir), path.Base(key))
|
||||
htmlCachePath = filepath.Join(cacheDir, filepath.FromSlash(keyDir), path.Base(key))
|
||||
} else {
|
||||
htmlCachePath = filepath.Join(cacheDir, path.Base(key))
|
||||
}
|
||||
if _, err := os.Stat(htmlCachePath); err == nil {
|
||||
return htmlCachePath, nil // 缓存命中
|
||||
}
|
||||
os.RemoveAll(cacheDir)
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return "", fmt.Errorf("创建缓存目录失败: %w", err)
|
||||
}
|
||||
tmpDir := cacheDir
|
||||
|
||||
htmlLocalPath := htmlCachePath
|
||||
if err := os.MkdirAll(filepath.Dir(htmlLocalPath), 0755); err != nil {
|
||||
os.RemoveAll(tmpDir)
|
||||
return "", fmt.Errorf("创建目录失败: %w", err)
|
||||
}
|
||||
} else {
|
||||
htmlLocalPath = filepath.Join(tmpDir, path.Base(key))
|
||||
}
|
||||
|
||||
// 2. 下载 HTML
|
||||
f, err := os.Create(htmlLocalPath)
|
||||
@@ -697,7 +708,7 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
|
||||
if err != nil {
|
||||
return htmlLocalPath, nil // HTML 已下载,资源解析失败不影响
|
||||
}
|
||||
resources := filesystem.ExtractHtmlResources(string(htmlContent))
|
||||
resources := common.ExtractHtmlResources(string(htmlContent))
|
||||
|
||||
// 4. 下载资源
|
||||
htmlOssDir := keyDir
|
||||
@@ -717,8 +728,11 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
|
||||
}
|
||||
}
|
||||
|
||||
// 绝对路径资源:串行嗅探 siteRoot,相对路径:并行下载
|
||||
type relativeTask struct{ ossKey, localPath string }
|
||||
var relativeTasks []relativeTask
|
||||
for _, resPath := range resources {
|
||||
if filesystem.ShouldSkipResource(resPath) {
|
||||
if common.ShouldSkipResource(resPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -743,11 +757,19 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
|
||||
} else {
|
||||
ossKey = cleanPath
|
||||
}
|
||||
if downloadResource(c, ctx, ossKey, localPath) {
|
||||
recordDir(ossKey)
|
||||
relativeTasks = append(relativeTasks, relativeTask{ossKey, localPath})
|
||||
}
|
||||
}
|
||||
|
||||
// 相对路径资源并行下载(recordDir 需加锁,多 goroutine 并发回调)
|
||||
var mu sync.Mutex
|
||||
common.RunConcurrent(relativeTasks, common.SiteDownloadConcurrency, func(t relativeTask) {
|
||||
if downloadResource(c, ctx, t.ossKey, t.localPath) {
|
||||
mu.Lock()
|
||||
recordDir(t.ossKey)
|
||||
mu.Unlock()
|
||||
}
|
||||
})
|
||||
// 5. 补充下载已发现目录中的剩余文件(覆盖 webpack 动态 chunk 等)
|
||||
for _, dir := range discoveredDirs {
|
||||
supplementDir(c, ctx, dir, tmpDir, siteRoot)
|
||||
@@ -756,7 +778,6 @@ func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string,
|
||||
return htmlLocalPath, nil
|
||||
}
|
||||
|
||||
// resolveAbsoluteResourcePath 解析绝对路径资源,首次嗅探网站根,后续直接使用
|
||||
// resolveAndDownload 解析绝对路径并下载:首次嗅探网站根,后续直接使用
|
||||
func resolveAndDownload(c oss.OSSProvider, ctx context.Context, htmlOssDir string, cleanPath string, localPath string, siteRoot *string) string {
|
||||
if *siteRoot != "" {
|
||||
@@ -815,37 +836,48 @@ func downloadResource(c oss.OSSProvider, ctx context.Context, ossKey string, loc
|
||||
}
|
||||
|
||||
// supplementDir 补充下载远程目录中尚未下载的文件(只处理已知资源所在目录)
|
||||
// 数量与单文件大小上限见 common.MaxSupplementFiles / common.MaxSupplementFileSize
|
||||
func supplementDir(c oss.OSSProvider, ctx context.Context, remoteDir string, tmpDir string, siteRoot string) {
|
||||
prefix := remoteDir + "/"
|
||||
result, err := c.ListFiles(ctx, &oss.ListOptions{Prefix: prefix, MaxKeys: 200})
|
||||
var marker string
|
||||
count := 0
|
||||
for count < common.MaxSupplementFiles {
|
||||
result, err := c.ListFiles(ctx, &oss.ListOptions{Prefix: prefix, MaxKeys: 200, Marker: marker})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, f := range result.Files {
|
||||
if strings.HasSuffix(f.Key, "/") || f.Size == 0 {
|
||||
if strings.HasSuffix(f.Key, "/") || f.Size == 0 || f.Size > common.MaxSupplementFileSize {
|
||||
continue
|
||||
}
|
||||
if count >= common.MaxSupplementFiles {
|
||||
log.Printf("[站点下载] 补充扫描达到上限: count=%d", common.MaxSupplementFiles)
|
||||
return
|
||||
}
|
||||
localPath := filepath.Join(tmpDir, filepath.FromSlash(f.Key))
|
||||
if _, err := os.Stat(localPath); err == nil {
|
||||
continue
|
||||
}
|
||||
downloadResource(c, ctx, f.Key, localPath)
|
||||
if downloadResource(c, ctx, f.Key, localPath) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if !result.IsTruncated {
|
||||
return
|
||||
}
|
||||
marker = result.NextMarker
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadToTemp 下载文件到本地临时目录(带 SQLite 缓存)
|
||||
func (s *Service) DownloadToTemp(connID string, rawPath string) (string, error) {
|
||||
// 先获取文件元信息用于缓存键,确保远程文件变更时能淘汰旧缓存
|
||||
var fileSize int64
|
||||
var modTime string
|
||||
if info, err := s.GetFileInfo(connID, rawPath); err == nil {
|
||||
if sz, ok := info["size"].(int64); ok {
|
||||
fileSize = sz
|
||||
}
|
||||
if mt, ok := info["mod_time"].(string); ok {
|
||||
modTime = mt
|
||||
}
|
||||
// 获取文件元信息用于缓存键
|
||||
info, err := s.GetFileInfo(connID, rawPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("获取文件信息失败: %w", err)
|
||||
}
|
||||
fileSize, _ := info["size"].(int64)
|
||||
modTime, _ := info["mod_time"].(string)
|
||||
return storage.DownloadToTempCached("oss", connID, rawPath, fileSize, modTime, func() (string, error) {
|
||||
return s.downloadToTempDirect(connID, rawPath)
|
||||
})
|
||||
@@ -920,7 +952,7 @@ func toOssOperationResult(m map[string]interface{}, isDir bool) *filesystem.File
|
||||
Path: p,
|
||||
Name: name,
|
||||
Size: size,
|
||||
SizeStr: filesystem.FormatBytes(size),
|
||||
SizeStr: common.FormatBytesInt(size),
|
||||
IsDir: isDir,
|
||||
ModTime: modTime,
|
||||
}
|
||||
|
||||
+13
-19
@@ -240,7 +240,7 @@ func replaceExecutableFile(newFilePath, execPath string) error {
|
||||
}
|
||||
|
||||
// Unix-like: 直接替换
|
||||
if err := copyFile(newFilePath, execPath); err != nil {
|
||||
if err := common.CopyFile(newFilePath, execPath); err != nil {
|
||||
return fmt.Errorf("复制文件失败: %v", err)
|
||||
}
|
||||
return os.Chmod(execPath, 0755)
|
||||
@@ -256,13 +256,13 @@ func replaceExecutableFileWindows(newFilePath, execPath string) error {
|
||||
os.Remove(newExecPathTemp)
|
||||
|
||||
// 复制新文件到临时位置
|
||||
if err := copyFile(newFilePath, newExecPathTemp); err != nil {
|
||||
if err := common.CopyFile(newFilePath, newExecPathTemp); err != nil {
|
||||
return fmt.Errorf("复制新文件失败: %v", err)
|
||||
}
|
||||
|
||||
// 尝试重命名当前文件(如果失败,说明文件正在使用)
|
||||
if err := os.Rename(execPath, oldExecPath); err != nil {
|
||||
return nil
|
||||
return fmt.Errorf("当前文件正在使用,无法替换: %w", err)
|
||||
}
|
||||
|
||||
// 替换文件
|
||||
@@ -350,7 +350,7 @@ func rollbackFromBackup(backupPath string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return copyFile(backupPath, execPath)
|
||||
return common.CopyFile(backupPath, execPath)
|
||||
}
|
||||
|
||||
// BackupApplication 备份当前应用
|
||||
@@ -369,27 +369,21 @@ func BackupApplication() (string, error) {
|
||||
backupFileName := fmt.Sprintf("u-desk-backup-%s%s", timestamp, filepath.Ext(execPath))
|
||||
backupPath := filepath.Join(backupDir, backupFileName)
|
||||
|
||||
if err := copyFile(execPath, backupPath); err != nil {
|
||||
if err := common.CopyFile(execPath, backupPath); err != nil {
|
||||
return "", fmt.Errorf("复制文件失败: %v", err)
|
||||
}
|
||||
|
||||
return backupPath, nil
|
||||
}
|
||||
|
||||
// copyFile 复制文件
|
||||
func copyFile(src, dst string) error {
|
||||
sourceFile, err := os.Open(src)
|
||||
// ReplacePendingFile 启动时完成上次更新遗留的 exe 替换。
|
||||
// 此时进程刚启动,exe 文件无占用,可安全完成上次因文件占用未完成的替换,
|
||||
// 并清理残留的 .old 备份。
|
||||
func ReplacePendingFile() {
|
||||
execPath, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
return
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
destFile, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer destFile.Close()
|
||||
|
||||
_, err = destFile.ReadFrom(sourceFile)
|
||||
return err
|
||||
_ = replacePendingFile(execPath) // 完成遗留的 .new -> exe 替换
|
||||
os.Remove(execPath + ".old") // 清理残留备份
|
||||
}
|
||||
|
||||
+98
-2
@@ -11,11 +11,22 @@ import (
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// 连接建立全流程(拨号+认证+SFTP 子系统)的总超时预算
|
||||
const connectTimeout = 10 * time.Second
|
||||
|
||||
// 保活参数:周期发送 keepalive 并等待回复,超过 keepaliveReplyWait 未回复判定连接已死
|
||||
const (
|
||||
keepaliveInterval = 30 * time.Second
|
||||
keepaliveReplyWait = 30 * time.Second
|
||||
)
|
||||
|
||||
// Client SFTP 客户端封装(单连接)
|
||||
type Client struct {
|
||||
config *Config
|
||||
client *sftp.Client
|
||||
sshClient *ssh.Client
|
||||
stopKeep chan struct{} // 当前 SSH 连接保活循环的停止信号
|
||||
closed bool // 保活失败或显式关闭后置位,供取连接时剔除
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
@@ -73,6 +84,11 @@ func (m *Manager) Disconnect(host string, port int) {
|
||||
}
|
||||
}
|
||||
|
||||
// Evict 从连接池剔除连接(底层已断开,仅移除不重复关闭)
|
||||
func (m *Manager) Evict(connID string) {
|
||||
m.clients.Delete(connID)
|
||||
}
|
||||
|
||||
// Shutdown 关闭所有连接
|
||||
func (m *Manager) Shutdown() {
|
||||
m.clients.Range(func(key, value any) bool {
|
||||
@@ -84,7 +100,18 @@ func (m *Manager) Shutdown() {
|
||||
|
||||
// --- 内部 ---
|
||||
|
||||
// newClient 建立连接并启动保活循环
|
||||
func newClient(config *Config) (*Client, error) {
|
||||
c, err := buildClient(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.startKeepalive()
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// buildClient 拨号+认证+SFTP 子系统建立(全程受 connectTimeout 约束),不启动保活
|
||||
func buildClient(config *Config) (*Client, error) {
|
||||
sshConfig := &ssh.ClientConfig{
|
||||
Config: ssh.Config{
|
||||
KeyExchanges: []string{
|
||||
@@ -131,10 +158,13 @@ func newClient(config *Config) (*Client, error) {
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", config.Host, config.Port)
|
||||
sshConn, err := net.DialTimeout("tcp", addr, config.Timeout)
|
||||
// 拨号与认证、SFTP 子系统握手共享同一总超时预算,认证阶段无界挂起由 deadline 掐断
|
||||
deadline := time.Now().Add(connectTimeout)
|
||||
sshConn, err := net.DialTimeout("tcp", addr, connectTimeout)
|
||||
if err != nil {
|
||||
return nil, &ConnectionError{Op: "dial", Err: err}
|
||||
}
|
||||
sshConn.SetDeadline(deadline)
|
||||
|
||||
sshConnConn, chans, reqs, err := ssh.NewClientConn(sshConn, addr, sshConfig)
|
||||
if err != nil {
|
||||
@@ -143,7 +173,9 @@ func newClient(config *Config) (*Client, error) {
|
||||
}
|
||||
|
||||
sshClient := ssh.NewClient(sshConnConn, chans, reqs)
|
||||
|
||||
sftpClient, err := sftp.NewClient(sshClient)
|
||||
sshConn.SetDeadline(time.Time{})
|
||||
if err != nil {
|
||||
sshClient.Close()
|
||||
return nil, &ConnectionError{Op: "sftp_init", Err: err}
|
||||
@@ -156,6 +188,62 @@ func newClient(config *Config) (*Client, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// startKeepalive 启动当前 SSH 连接的保活循环(须持有 c.mu 或独占 Client 时调用)
|
||||
// 防止空闲连接被 NAT/防火墙杀掉;探测失败时关闭连接并置位 closed,唤醒所有阻塞中的操作
|
||||
func (c *Client) startKeepalive() {
|
||||
stop := make(chan struct{})
|
||||
sshClient := c.sshClient
|
||||
c.stopKeep = stop
|
||||
go func() {
|
||||
t := time.NewTicker(keepaliveInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
if !sshKeepalive(sshClient, keepaliveReplyWait) {
|
||||
c.markDead(sshClient)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// sshKeepalive 发送一次 keepalive 请求并限时等待回复,超时或出错视作连接已死
|
||||
func sshKeepalive(sshClient *ssh.Client, wait time.Duration) bool {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, err := sshClient.SendRequest("keepalive@openssh.com", true, nil)
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err == nil
|
||||
case <-time.After(wait):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// markDead 标记连接死亡并关闭底层资源
|
||||
// 仅当容器仍持有该 SSH 连接时生效,避免误杀 reconnect 换上的新连接
|
||||
func (c *Client) markDead(sshClient *ssh.Client) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.sshClient != sshClient {
|
||||
return
|
||||
}
|
||||
c.closeLocked()
|
||||
}
|
||||
|
||||
// IsClosed 连接是否已断开(保活失败或显式关闭)
|
||||
func (c *Client) IsClosed() bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.closed
|
||||
}
|
||||
|
||||
// IsHealthy 检查连接是否健康(先取引用再解锁,避免持锁做 I/O)
|
||||
func (c *Client) IsHealthy() bool {
|
||||
c.mu.Lock()
|
||||
@@ -204,7 +292,8 @@ func (c *Client) WithRetry(fn func(*sftp.Client) error) error {
|
||||
}
|
||||
|
||||
func (c *Client) reconnect() error {
|
||||
nc, err := newClient(c.config)
|
||||
// 用 buildClient 而非 newClient,避免保活循环绑定到即将丢弃的临时容器
|
||||
nc, err := buildClient(c.config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -213,6 +302,8 @@ func (c *Client) reconnect() error {
|
||||
c.closeLocked()
|
||||
c.client = nc.client
|
||||
c.sshClient = nc.sshClient
|
||||
c.closed = false
|
||||
c.startKeepalive()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -223,6 +314,10 @@ func (c *Client) Close() {
|
||||
}
|
||||
|
||||
func (c *Client) closeLocked() {
|
||||
if c.stopKeep != nil {
|
||||
close(c.stopKeep)
|
||||
c.stopKeep = nil
|
||||
}
|
||||
if c.client != nil {
|
||||
c.client.Close()
|
||||
c.client = nil
|
||||
@@ -231,6 +326,7 @@ func (c *Client) closeLocked() {
|
||||
c.sshClient.Close()
|
||||
c.sshClient = nil
|
||||
}
|
||||
c.closed = true
|
||||
}
|
||||
|
||||
// RunCommand 通过 SSH Session 执行远程命令,返回 stdout
|
||||
|
||||
+284
-15
@@ -1,8 +1,12 @@
|
||||
package sftp
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
@@ -12,6 +16,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"u-desk/internal/common"
|
||||
"u-desk/internal/filesystem"
|
||||
"u-desk/internal/storage"
|
||||
|
||||
@@ -84,7 +89,7 @@ func (s *Service) ReadFile(connID string, filePath string) (string, error) {
|
||||
fi, e := sc.Stat(filePath)
|
||||
if e != nil { return e }
|
||||
if fi.Size() > maxSize {
|
||||
return fmt.Errorf("文件过大 (%s),超过 %d 限制", filesystem.FormatBytes(fi.Size()), maxSize)
|
||||
return fmt.Errorf("文件过大 (%s),超过 %d 限制", common.FormatBytesInt(fi.Size()), maxSize)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
@@ -105,7 +110,7 @@ func (s *Service) ReadFile(connID string, filePath string) (string, error) {
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("读取文件失败: %w", err)
|
||||
}
|
||||
return filesystem.BytesToString(data), nil
|
||||
return common.BytesToString(data), nil
|
||||
}
|
||||
|
||||
func (s *Service) WriteFile(connID string, filePath string, content string) error {
|
||||
@@ -168,7 +173,7 @@ func (s *Service) GetFileInfo(connID string, filePath string) (map[string]interf
|
||||
"name": info.Name(),
|
||||
"path": toUnixPath(filePath),
|
||||
"size": info.Size(),
|
||||
"size_str": filesystem.FormatBytes(info.Size()),
|
||||
"size_str": common.FormatBytesInt(info.Size()),
|
||||
"is_dir": info.IsDir(),
|
||||
"mod_time": info.ModTime().Format("2006-01-02 15:04:05"),
|
||||
"mode": info.Mode().String(),
|
||||
@@ -209,7 +214,10 @@ func (s *Service) CreateFile(connID string, filePath string) (*filesystem.FileOp
|
||||
return nil, fmt.Errorf("创建文件失败: %w", err)
|
||||
}
|
||||
|
||||
infoMap, _ := s.GetFileInfo(connID, filePath)
|
||||
infoMap, err := s.GetFileInfo(connID, filePath)
|
||||
if err != nil {
|
||||
slog.Warn("创建后获取详情失败", "path", filePath, "error", err)
|
||||
}
|
||||
return toFileOperationResult(infoMap, false), nil
|
||||
}
|
||||
|
||||
@@ -291,7 +299,7 @@ func (s *Service) downloadToTempDirect(connID string, remotePath string) (string
|
||||
fi, e := sc.Stat(remotePath)
|
||||
if e != nil { return e }
|
||||
if fi.Size() > maxPreviewSize {
|
||||
return fmt.Errorf("预览文件过大: %s", filesystem.FormatBytes(fi.Size()))
|
||||
return fmt.Errorf("预览文件过大: %s", common.FormatBytesInt(fi.Size()))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
@@ -345,11 +353,20 @@ func (s *Service) DownloadSiteForPreview(connID string, remotePath string) (stri
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 1. 创建临时目录
|
||||
tmpDir, err := os.MkdirTemp("", "udesk-sftp-site-*")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("创建临时目录失败: %w", err)
|
||||
// 1. 缓存检查:用确定性路径,命中直接返回
|
||||
htmlInfo, _ := s.GetFileInfo(connID, remotePath)
|
||||
htmlSize, _ := htmlInfo["size"].(int64)
|
||||
htmlModTime, _ := htmlInfo["mod_time"].(string)
|
||||
cacheDir := common.SiteCacheDir("sftp", connID, remotePath, htmlSize, htmlModTime)
|
||||
htmlCachePath := filepath.Join(cacheDir, filepath.FromSlash(path.Dir(remotePath)), path.Base(remotePath))
|
||||
if _, err := os.Stat(htmlCachePath); err == nil {
|
||||
return htmlCachePath, nil // 缓存命中
|
||||
}
|
||||
os.RemoveAll(cacheDir) // 清理旧缓存
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return "", fmt.Errorf("创建缓存目录失败: %w", err)
|
||||
}
|
||||
tmpDir := cacheDir
|
||||
|
||||
// 2. 确定远程网站根目录(从 HTML 路径推断)
|
||||
keyDir := path.Dir(remotePath)
|
||||
@@ -379,7 +396,7 @@ func (s *Service) DownloadSiteForPreview(connID string, remotePath string) (stri
|
||||
if err != nil {
|
||||
return htmlLocalPath, nil
|
||||
}
|
||||
resources := filesystem.ExtractHtmlResources(string(htmlContent))
|
||||
resources := common.ExtractHtmlResources(string(htmlContent))
|
||||
|
||||
// 5. 下载静态引用资源(嗅探网站根)
|
||||
htmlRemoteDir := keyDir
|
||||
@@ -399,8 +416,11 @@ func (s *Service) DownloadSiteForPreview(connID string, remotePath string) (stri
|
||||
}
|
||||
}
|
||||
|
||||
// 绝对路径资源:串行嗅探 siteRoot,相对路径:并行下载
|
||||
type relativeTask struct{ remoteKey, localPath string }
|
||||
var relativeTasks []relativeTask
|
||||
for _, resPath := range resources {
|
||||
if filesystem.ShouldSkipResource(resPath) {
|
||||
if common.ShouldSkipResource(resPath) {
|
||||
continue
|
||||
}
|
||||
isAbsolute := strings.HasPrefix(resPath, "/")
|
||||
@@ -418,11 +438,19 @@ func (s *Service) DownloadSiteForPreview(connID string, remotePath string) (stri
|
||||
}
|
||||
} else {
|
||||
remoteKey := path.Join(htmlRemoteDir, cleanPath)
|
||||
if s.sftpTryDownload(c, remoteKey, localPath) {
|
||||
recordDir(remoteKey)
|
||||
relativeTasks = append(relativeTasks, relativeTask{remoteKey, localPath})
|
||||
}
|
||||
}
|
||||
|
||||
// 相对路径资源并行下载(recordDir 需加锁,多 goroutine 并发回调)
|
||||
var mu sync.Mutex
|
||||
common.RunConcurrent(relativeTasks, common.SiteDownloadConcurrency, func(t relativeTask) {
|
||||
if s.sftpTryDownload(c, t.remoteKey, t.localPath) {
|
||||
mu.Lock()
|
||||
recordDir(t.remoteKey)
|
||||
mu.Unlock()
|
||||
}
|
||||
})
|
||||
|
||||
// 6. 补充下载已发现目录中的剩余文件(覆盖动态 chunk 等)
|
||||
for _, dir := range discoveredDirs {
|
||||
@@ -494,6 +522,7 @@ func sftpResolveAndDownload(s *Service, c *Client, htmlDir string, cleanPath str
|
||||
}
|
||||
|
||||
// sftpSupplementDir 补充下载远程目录中尚未下载的文件(只处理已知资源所在目录)
|
||||
// 数量与单文件大小上限见 common.MaxSupplementFiles / common.MaxSupplementFileSize
|
||||
func sftpSupplementDir(s *Service, c *Client, remoteDir string, tmpDir string, siteRoot string) {
|
||||
var entries []fs.FileInfo
|
||||
err := c.WithRetry(func(sc *sftpclient.Client) error {
|
||||
@@ -504,19 +533,254 @@ func sftpSupplementDir(s *Service, c *Client, remoteDir string, tmpDir string, s
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
count := 0
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || entry.Size() == 0 {
|
||||
if entry.IsDir() || entry.Size() == 0 || entry.Size() > common.MaxSupplementFileSize {
|
||||
continue
|
||||
}
|
||||
if count >= common.MaxSupplementFiles {
|
||||
log.Printf("[站点下载] 补充扫描达到上限: count=%d", common.MaxSupplementFiles)
|
||||
break
|
||||
}
|
||||
fullPath := path.Join(remoteDir, entry.Name())
|
||||
localPath := filepath.Join(tmpDir, filepath.FromSlash(fullPath))
|
||||
if _, err := os.Stat(localPath); err == nil {
|
||||
continue
|
||||
}
|
||||
s.sftpTryDownload(c, fullPath, localPath)
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadToFile 下载远程文件/目录到用户指定的本地目录(流式,无大小上限),返回实际落地路径
|
||||
// 目标重名自动追加「 - 副本」后缀;onProgress 按累计字节回调(name 为当前文件名)
|
||||
func (s *Service) DownloadToFile(connID, remotePath, localDstDir string, onProgress func(fileName string, copied, total int64)) (string, error) {
|
||||
c, err := s.getClient(connID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var info fs.FileInfo
|
||||
err = c.WithRetry(func(sc *sftpclient.Client) error {
|
||||
var e error
|
||||
info, e = sc.Stat(remotePath)
|
||||
return e
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("获取远程文件信息失败: %w", err)
|
||||
}
|
||||
|
||||
// 单文件:直接流式落地
|
||||
if !info.IsDir() {
|
||||
dstPath := filesystem.UniquePath(filepath.Join(localDstDir, path.Base(remotePath)))
|
||||
var copied int64
|
||||
if err := s.downloadFileTo(c, remotePath, dstPath, &copied, info.Size(), onProgress); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dstPath, nil
|
||||
}
|
||||
|
||||
// 目录:先收集远端文件清单并统计总大小
|
||||
type remoteEntry struct {
|
||||
remote, local string
|
||||
size int64
|
||||
}
|
||||
var entries []remoteEntry
|
||||
var total int64
|
||||
var walk func(dir, localDir string) error
|
||||
walk = func(dir, localDir string) error {
|
||||
var list []fs.FileInfo
|
||||
err := c.WithRetry(func(sc *sftpclient.Client) error {
|
||||
var e error
|
||||
list, e = sc.ReadDir(dir)
|
||||
return e
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, fi := range list {
|
||||
rp := path.Join(dir, fi.Name())
|
||||
lp := filepath.Join(localDir, fi.Name())
|
||||
if fi.IsDir() {
|
||||
if err := walk(rp, lp); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if fi.Mode().IsRegular() {
|
||||
entries = append(entries, remoteEntry{rp, lp, fi.Size()})
|
||||
total += fi.Size()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
dstDir := filesystem.UniquePath(filepath.Join(localDstDir, path.Base(remotePath)))
|
||||
if err := walk(remotePath, dstDir); err != nil {
|
||||
return "", fmt.Errorf("遍历远程目录失败: %w", err)
|
||||
}
|
||||
|
||||
var copied int64
|
||||
for _, e := range entries {
|
||||
if err := s.downloadFileTo(c, e.remote, e.local, &copied, total, onProgress); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return dstDir, nil
|
||||
}
|
||||
|
||||
// downloadFileTo 流式下载单个远程文件,进度按累计字节回调
|
||||
func (s *Service) downloadFileTo(c *Client, remotePath, localPath string, copied *int64, total int64, onProgress func(fileName string, copied, total int64)) error {
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
|
||||
return fmt.Errorf("创建本地目录失败: %w", err)
|
||||
}
|
||||
return c.WithRetry(func(sc *sftpclient.Client) error {
|
||||
src, err := sc.Open(remotePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
dst, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dst.Close()
|
||||
var reader io.Reader = src
|
||||
if onProgress != nil {
|
||||
reader = &common.CountingReader{R: src, OnN: func(n int64) {
|
||||
*copied += n
|
||||
onProgress(path.Base(remotePath), *copied, total)
|
||||
}}
|
||||
}
|
||||
_, err = io.Copy(dst, reader)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// UploadFromFile 上传本地文件/目录到远程目录(流式),远端重名自动追加副本后缀
|
||||
func (s *Service) UploadFromFile(connID, localPath, remoteDir string, onProgress func(fileName string, copied, total int64)) error {
|
||||
c, err := s.getClient(connID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info, err := os.Stat(localPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("本地路径不存在: %w", err)
|
||||
}
|
||||
|
||||
// 单文件:直接流式上传
|
||||
if !info.IsDir() {
|
||||
remotePath, err := s.uniqueRemotePath(c, path.Join(remoteDir, filepath.Base(localPath)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var copied int64
|
||||
return s.uploadFileTo(c, localPath, remotePath, &copied, info.Size(), onProgress)
|
||||
}
|
||||
|
||||
// 目录:本地清单 + 总大小(相对根路径,远端落地根确定后再拼完整路径)
|
||||
type localEntry struct {
|
||||
local, rel string
|
||||
size int64
|
||||
}
|
||||
var files []localEntry
|
||||
var total int64
|
||||
err = filepath.WalkDir(localPath, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.Type().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
fi, e := d.Info()
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
rel, e := filepath.Rel(localPath, p)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
files = append(files, localEntry{p, filepath.ToSlash(rel), fi.Size()})
|
||||
total += fi.Size()
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("遍历本地目录失败: %w", err)
|
||||
}
|
||||
|
||||
remoteRoot, err := s.uniqueRemotePath(c, path.Join(remoteDir, filepath.Base(localPath)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var copied int64
|
||||
for _, f := range files {
|
||||
if err := s.uploadFileTo(c, f.local, path.Join(remoteRoot, f.rel), &copied, total, onProgress); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadFileTo 流式上传单个本地文件(自动创建远端父目录),进度按累计字节回调
|
||||
func (s *Service) uploadFileTo(c *Client, localPath, remotePath string, copied *int64, total int64, onProgress func(fileName string, copied, total int64)) error {
|
||||
return c.WithRetry(func(sc *sftpclient.Client) error {
|
||||
if err := sc.MkdirAll(path.Dir(remotePath)); err != nil {
|
||||
return err
|
||||
}
|
||||
src, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
dst, err := sc.Create(remotePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dst.Close()
|
||||
var reader io.Reader = src
|
||||
if onProgress != nil {
|
||||
reader = &common.CountingReader{R: src, OnN: func(n int64) {
|
||||
*copied += n
|
||||
onProgress(filepath.Base(localPath), *copied, total)
|
||||
}}
|
||||
}
|
||||
_, err = io.Copy(dst, reader)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// uniqueRemotePath 远端重名时追加「 - 副本」「 - 副本 (2)」后缀(扩展名前插入)
|
||||
// 命名循环复用 common.UniqueName(与本地 UniquePath 同源)
|
||||
func (s *Service) uniqueRemotePath(c *Client, remotePath string) (string, error) {
|
||||
dir := path.Dir(remotePath)
|
||||
unique, err := common.UniqueName(path.Base(remotePath), func(candidate string) (bool, error) {
|
||||
return s.remoteExists(c, path.Join(dir, candidate))
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path.Join(dir, unique), nil
|
||||
}
|
||||
|
||||
// remoteExists 判断远程路径是否存在(区分不存在与网络错误)
|
||||
func (s *Service) remoteExists(c *Client, p string) (bool, error) {
|
||||
var notExist bool
|
||||
err := c.WithRetry(func(sc *sftpclient.Client) error {
|
||||
_, e := sc.Stat(p)
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(e, fs.ErrNotExist) {
|
||||
notExist = true
|
||||
return nil
|
||||
}
|
||||
return e
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return !notExist, nil
|
||||
}
|
||||
|
||||
// GetCommonPaths 返回 SFTP 远程主机常用路径
|
||||
func (s *Service) GetCommonPaths(connID string) (map[string]string, error) {
|
||||
c := s.manager.GetClient(connID)
|
||||
@@ -676,6 +940,11 @@ func (s *Service) getClient(connID string) (*Client, error) {
|
||||
if c == nil {
|
||||
return nil, fmt.Errorf("SFTP 连接不存在: %s", connID)
|
||||
}
|
||||
if c.IsClosed() {
|
||||
// 保活失败或显式断开置位,剔除死连接避免后续操作继续挂在上面
|
||||
s.manager.Evict(connID)
|
||||
return nil, fmt.Errorf("SFTP 连接已断开,请重新连接: %s", connID)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
@@ -694,7 +963,7 @@ func toFileOperationResult(m map[string]interface{}, isDir bool) *filesystem.Fil
|
||||
Path: p,
|
||||
Name: name,
|
||||
Size: size,
|
||||
SizeStr: filesystem.FormatBytes(size),
|
||||
SizeStr: common.FormatBytesInt(size),
|
||||
IsDir: isDir,
|
||||
ModTime: modTime,
|
||||
Mode: mode,
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"u-desk/internal/common"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"u-desk/internal/storage/models"
|
||||
@@ -129,7 +131,7 @@ func DownloadToTempCached(transport, connID, remotePath string, fileSize int64,
|
||||
// 移动文件到确定性路径
|
||||
if err := os.Rename(tempPath, deterministicPath); err != nil {
|
||||
// Rename 可能跨卷失败,尝试 Copy+Delete
|
||||
if copyFile(tempPath, deterministicPath) != nil {
|
||||
if common.CopyFile(tempPath, deterministicPath) != nil {
|
||||
SaveCache(transport, connID, remotePath, fileSize, modTime, tempPath)
|
||||
return tempPath, nil
|
||||
}
|
||||
@@ -166,23 +168,37 @@ func deterministicCachePath(transport, connID, remotePath string, fileSize int64
|
||||
return filepath.Join(cacheTempDir, fileName), nil
|
||||
}
|
||||
|
||||
// copyFile 复制文件内容
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
// CleanupSiteCacheDirs 清理超过 24h 的站点预览缓存目录
|
||||
// os.TempDir() 下两个前缀是旧版 MkdirTemp 随机目录残留(sftp 与 oss 各一种)
|
||||
func CleanupSiteCacheDirs() {
|
||||
cutoff := time.Now().Add(-downloadCacheTTL)
|
||||
cleaned := cleanupExpiredDirs(cacheTempDir, "site-", cutoff)
|
||||
cleaned += cleanupExpiredDirs(os.TempDir(), "udesk-site-", cutoff)
|
||||
cleaned += cleanupExpiredDirs(os.TempDir(), "udesk-sftp-site-", cutoff)
|
||||
if cleaned > 0 {
|
||||
slog.Info("清理过期站点缓存目录", "count", cleaned)
|
||||
}
|
||||
defer in.Close()
|
||||
}
|
||||
|
||||
out, err := os.Create(dst)
|
||||
// cleanupExpiredDirs 删除 dir 下修改时间早于 cutoff 的 prefix 前缀子目录,返回清理数量
|
||||
func cleanupExpiredDirs(dir, prefix string, cutoff time.Time) int {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
return 0
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
if _, err := out.ReadFrom(in); err != nil {
|
||||
os.Remove(dst)
|
||||
return err
|
||||
cleaned := 0
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() || !strings.HasPrefix(e.Name(), prefix) {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if info.ModTime().Before(cutoff) {
|
||||
os.RemoveAll(filepath.Join(dir, e.Name()))
|
||||
cleaned++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return cleaned
|
||||
}
|
||||
|
||||
@@ -61,6 +61,8 @@ func InitFast() (*gorm.DB, error) {
|
||||
db.Exec("UPDATE connection_profiles SET provider = type, type = 'oss' WHERE type IN ('qiniu', 'aliyun')")
|
||||
// 为旧 BGM 播放列表补充 profile_id(找第一个 OSS profile)
|
||||
db.Exec("UPDATE bgm_playlist SET profile_id = (SELECT CAST(id AS VARCHAR) FROM connection_profiles WHERE type = 'oss' LIMIT 1) WHERE (profile_id = '' OR profile_id IS NULL) AND path NOT LIKE '%:'")
|
||||
// 清理历史版本 id 断链重复 INSERT 堆积的重复服务器配置,保留每组 id 最小的一行
|
||||
db.Exec("DELETE FROM connection_profiles WHERE id NOT IN (SELECT MIN(id) FROM connection_profiles GROUP BY name, type, host, port, bucket)")
|
||||
globalDB = db
|
||||
})
|
||||
if initErr != nil {
|
||||
|
||||
Reference in New Issue
Block a user