lxy
6ea9d9ac99
- watcher握手期伪触发致幻影本地列表，根因修复 - 连接探测2.5s超时竞速，握手10s预算 - 30s保活自动掐死半开连接，取连接时剔除已断开 - 副本命名与进度计数抽公共，下载缓存清理去重
960 lines
25 KiB
Go
960 lines
25 KiB
Go
package ossdrv
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/base64"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"path"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"u-desk/internal/common"
|
||
"u-desk/internal/filesystem"
|
||
"u-desk/internal/oss"
|
||
"u-desk/internal/oss/aliyun"
|
||
"u-desk/internal/oss/qiniu"
|
||
"u-desk/internal/storage"
|
||
)
|
||
|
||
// accountCredentials 账户级凭据
|
||
type accountCredentials struct {
|
||
Provider string
|
||
AccessKey string
|
||
SecretKey string
|
||
Endpoint string
|
||
}
|
||
|
||
// Manager OSS 连接管理器（两级：账户 + 桶级客户端缓存）
|
||
type Manager struct {
|
||
accounts sync.Map // map[string]*accountCredentials key=provider
|
||
clients sync.Map // map[string]oss.OSSProvider key="provider:bucket"
|
||
bucketRegions sync.Map // map[string]string key="provider:bucket" → region
|
||
}
|
||
|
||
var globalManager = &Manager{}
|
||
|
||
func GetManager() *Manager { return globalManager }
|
||
|
||
// Connect 建立账户级连接（验证凭据通过 ListBuckets，同时缓存桶区域）
|
||
func (m *Manager) Connect(provider, accessKey, secretKey, endpoint string) error {
|
||
var entries []oss.BucketEntry
|
||
var err error
|
||
|
||
switch provider {
|
||
case "qiniu":
|
||
entries, err = qiniu.ListBuckets(accessKey, secretKey)
|
||
if err != nil {
|
||
return fmt.Errorf("七牛云连接失败: %w", err)
|
||
}
|
||
case "aliyun":
|
||
entries, err = aliyun.ListBuckets(accessKey, secretKey, endpoint)
|
||
if err != nil {
|
||
return fmt.Errorf("阿里云连接失败: %w", err)
|
||
}
|
||
default:
|
||
return fmt.Errorf("不支持的 OSS 提供商: %s", provider)
|
||
}
|
||
|
||
// 连接时立即缓存桶区域，避免后续操作因缺少 region 使用默认区域
|
||
for _, e := range entries {
|
||
if e.Region != "" {
|
||
m.bucketRegions.Store(provider+":"+e.Name, e.Region)
|
||
}
|
||
}
|
||
|
||
m.accounts.Store(provider, &accountCredentials{
|
||
Provider: provider,
|
||
AccessKey: accessKey,
|
||
SecretKey: secretKey,
|
||
Endpoint: endpoint,
|
||
})
|
||
return nil
|
||
}
|
||
|
||
// getOrCreateBucketClient 懒创建桶级 OSSProvider
|
||
func (m *Manager) getOrCreateBucketClient(provider, bucket, region string) (oss.OSSProvider, error) {
|
||
key := provider + ":" + bucket
|
||
if v, ok := m.clients.Load(key); ok {
|
||
return v.(oss.OSSProvider), nil
|
||
}
|
||
|
||
cred, ok := m.accounts.Load(provider)
|
||
if !ok {
|
||
return nil, fmt.Errorf("OSS 账户未连接: %s", provider)
|
||
}
|
||
c := cred.(*accountCredentials)
|
||
|
||
// 如果未传 region，从缓存取；仍为空则主动探测
|
||
if region == "" {
|
||
if v, ok := m.bucketRegions.Load(key); ok {
|
||
region = v.(string)
|
||
} else {
|
||
region = m.detectBucketRegion(provider, bucket, c)
|
||
if region != "" {
|
||
m.bucketRegions.Store(key, region)
|
||
}
|
||
}
|
||
}
|
||
|
||
var client oss.OSSProvider
|
||
var err error
|
||
|
||
switch provider {
|
||
case "qiniu":
|
||
client, err = qiniu.NewClient(&qiniu.Config{
|
||
AccessKey: c.AccessKey,
|
||
SecretKey: c.SecretKey,
|
||
Bucket: bucket,
|
||
Region: region,
|
||
UseHTTPS: true,
|
||
})
|
||
case "aliyun":
|
||
client, err = aliyun.NewClient(&aliyun.Config{
|
||
AccessKeyID: c.AccessKey,
|
||
AccessKeySecret: c.SecretKey,
|
||
Bucket: bucket,
|
||
Region: region,
|
||
Endpoint: c.Endpoint,
|
||
UseHTTPS: true,
|
||
})
|
||
default:
|
||
return nil, fmt.Errorf("不支持的提供商: %s", provider)
|
||
}
|
||
|
||
if err != nil {
|
||
return nil, fmt.Errorf("创建桶客户端失败: %w", err)
|
||
}
|
||
|
||
m.clients.Store(key, client)
|
||
return client, nil
|
||
}
|
||
|
||
// detectBucketRegion 主动探测桶区域（缓存未命中时调用）
|
||
func (m *Manager) detectBucketRegion(provider, bucket string, c *accountCredentials) string {
|
||
switch provider {
|
||
case "aliyun":
|
||
entries, err := aliyun.ListBuckets(c.AccessKey, c.SecretKey, c.Endpoint)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
for _, e := range entries {
|
||
key := provider + ":" + e.Name
|
||
if e.Region != "" {
|
||
m.bucketRegions.Store(key, e.Region)
|
||
}
|
||
if e.Name == bucket {
|
||
return e.Region
|
||
}
|
||
}
|
||
case "qiniu":
|
||
entries, err := qiniu.ListBuckets(c.AccessKey, c.SecretKey)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
for _, e := range entries {
|
||
key := provider + ":" + e.Name
|
||
if e.Region != "" {
|
||
m.bucketRegions.Store(key, e.Region)
|
||
}
|
||
if e.Name == bucket {
|
||
return e.Region
|
||
}
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// GetClient 获取已有的桶级客户端
|
||
func (m *Manager) GetClient(provider, bucket string) oss.OSSProvider {
|
||
if v, ok := m.clients.Load(provider + ":" + bucket); ok {
|
||
return v.(oss.OSSProvider)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Disconnect 关闭账户及所有桶级客户端
|
||
func (m *Manager) Disconnect(provider string) {
|
||
m.accounts.Delete(provider)
|
||
prefix := provider + ":"
|
||
m.clients.Range(func(key, value any) bool {
|
||
if strings.HasPrefix(key.(string), prefix) {
|
||
value.(oss.OSSProvider).Close()
|
||
m.clients.Delete(key)
|
||
}
|
||
return true
|
||
})
|
||
m.bucketRegions.Range(func(key, value any) bool {
|
||
if strings.HasPrefix(key.(string), prefix) {
|
||
m.bucketRegions.Delete(key)
|
||
}
|
||
return true
|
||
})
|
||
}
|
||
|
||
// Shutdown 关闭所有连接
|
||
func (m *Manager) Shutdown() {
|
||
m.clients.Range(func(key, value any) bool {
|
||
value.(oss.OSSProvider).Close()
|
||
m.clients.Delete(key)
|
||
return true
|
||
})
|
||
m.accounts.Range(func(key, value any) bool {
|
||
m.accounts.Delete(key)
|
||
return true
|
||
})
|
||
}
|
||
|
||
// Service OSS 文件操作服务
|
||
type Service struct {
|
||
manager *Manager
|
||
}
|
||
|
||
func NewService() *Service {
|
||
return &Service{manager: GetManager()}
|
||
}
|
||
|
||
func (s *Service) GetManager() *Manager { return s.manager }
|
||
|
||
// parseBucketPath 解析路径中的桶名和对象键
|
||
// "/my-bucket/photos/img.jpg" → bucket="my-bucket", key="photos/img.jpg"
|
||
func parseBucketPath(rawPath string) (bucket, key string) {
|
||
rawPath = strings.TrimPrefix(rawPath, "/")
|
||
if rawPath == "" {
|
||
return "", ""
|
||
}
|
||
parts := strings.SplitN(rawPath, "/", 2)
|
||
bucket = parts[0]
|
||
if len(parts) > 1 {
|
||
key = parts[1]
|
||
}
|
||
return
|
||
}
|
||
|
||
// listBuckets 列出所有桶
|
||
func (s *Service) listBuckets(provider string) ([]map[string]interface{}, error) {
|
||
cred, ok := s.manager.accounts.Load(provider)
|
||
if !ok {
|
||
return nil, fmt.Errorf("OSS 账户未连接: %s", provider)
|
||
}
|
||
c := cred.(*accountCredentials)
|
||
|
||
var entries []oss.BucketEntry
|
||
var err error
|
||
|
||
switch provider {
|
||
case "qiniu":
|
||
entries, err = qiniu.ListBuckets(c.AccessKey, c.SecretKey)
|
||
case "aliyun":
|
||
entries, err = aliyun.ListBuckets(c.AccessKey, c.SecretKey, c.Endpoint)
|
||
default:
|
||
return nil, fmt.Errorf("不支持的提供商: %s", provider)
|
||
}
|
||
if err != nil {
|
||
return nil, fmt.Errorf("列举存储桶失败: %w", err)
|
||
}
|
||
|
||
// 缓存桶区域信息
|
||
for _, e := range entries {
|
||
if e.Region != "" {
|
||
s.manager.bucketRegions.Store(provider+":"+e.Name, e.Region)
|
||
}
|
||
}
|
||
|
||
items := make([]map[string]interface{}, len(entries))
|
||
for i, e := range entries {
|
||
items[i] = map[string]interface{}{
|
||
"name": e.Name,
|
||
"path": "/" + e.Name,
|
||
"is_dir": true,
|
||
"is_bucket": true,
|
||
"size": int64(0),
|
||
}
|
||
}
|
||
return items, nil
|
||
}
|
||
|
||
// ListDir 列出目录内容
|
||
func (s *Service) ListDir(connID string, prefix string) ([]map[string]interface{}, error) {
|
||
prefix = strings.TrimPrefix(prefix, "/")
|
||
|
||
// 根目录 → 列出所有桶
|
||
if prefix == "" {
|
||
return s.listBuckets(connID)
|
||
}
|
||
|
||
// 解析桶名和对象前缀
|
||
bucket, objectPrefix := parseBucketPath(prefix)
|
||
if bucket == "" {
|
||
return s.listBuckets(connID)
|
||
}
|
||
|
||
if objectPrefix != "" && !strings.HasSuffix(objectPrefix, "/") {
|
||
objectPrefix += "/"
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
ctx := context.Background()
|
||
result, err := c.ListFiles(ctx, &oss.ListOptions{
|
||
Prefix: objectPrefix,
|
||
Delimiter: "/",
|
||
MaxKeys: 1000,
|
||
})
|
||
if err != nil {
|
||
return nil, fmt.Errorf("列举文件失败: %w", err)
|
||
}
|
||
|
||
items := make([]map[string]interface{}, 0, len(result.Files)+len(result.Prefixes))
|
||
bucketPrefix := "/" + bucket + "/"
|
||
|
||
for _, p := range result.Prefixes {
|
||
name := strings.TrimSuffix(strings.TrimPrefix(p, objectPrefix), "/")
|
||
if name == "" {
|
||
continue
|
||
}
|
||
items = append(items, map[string]interface{}{
|
||
"name": name,
|
||
"path": bucketPrefix + p,
|
||
"is_dir": true,
|
||
"size": int64(0),
|
||
})
|
||
}
|
||
|
||
for _, f := range result.Files {
|
||
if strings.HasSuffix(f.Key, "/") && f.Size == 0 {
|
||
continue
|
||
}
|
||
items = append(items, map[string]interface{}{
|
||
"name": path.Base(f.Key),
|
||
"path": bucketPrefix + f.Key,
|
||
"is_dir": false,
|
||
"size": f.Size,
|
||
"mod_time": f.LastModified.Format("2006-01-02 15:04:05"),
|
||
})
|
||
}
|
||
|
||
return items, nil
|
||
}
|
||
|
||
// ReadFile 读取文件内容
|
||
func (s *Service) ReadFile(connID string, rawPath string) (string, error) {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return "", fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
const maxSize int64 = 10 << 20
|
||
ctx := context.Background()
|
||
|
||
info, err := c.GetFileInfo(ctx, key)
|
||
if err != nil {
|
||
return "", fmt.Errorf("获取文件信息失败: %w", err)
|
||
}
|
||
if info.Size > maxSize {
|
||
return "", fmt.Errorf("文件过大 (%s)，超过 %d 限制", common.FormatBytesInt(info.Size), maxSize)
|
||
}
|
||
|
||
var buf bytes.Buffer
|
||
if err := c.Download(ctx, key, &buf); err != nil {
|
||
return "", fmt.Errorf("读取文件失败: %w", err)
|
||
}
|
||
return common.BytesToString(buf.Bytes()), nil
|
||
}
|
||
|
||
// WriteFile 写入文件内容
|
||
func (s *Service) WriteFile(connID string, rawPath string, content string) error {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
_, err = c.Upload(context.Background(), key, strings.NewReader(content), nil)
|
||
if err != nil {
|
||
return fmt.Errorf("写入文件失败: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// WriteBase64File 写入 base64 编码的二进制文件
|
||
func (s *Service) WriteBase64File(connID string, rawPath string, base64Content string) error {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
data, err := base64.StdEncoding.DecodeString(base64Content)
|
||
if err != nil {
|
||
return fmt.Errorf("base64 解码失败: %w", err)
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
_, err = c.Upload(context.Background(), key, bytes.NewReader(data), nil)
|
||
if err != nil {
|
||
return fmt.Errorf("写入文件失败: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// GetFileInfo 获取文件信息
|
||
func (s *Service) GetFileInfo(connID string, rawPath string) (map[string]interface{}, error) {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return nil, fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
info, err := c.GetFileInfo(context.Background(), key)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("获取文件信息失败: %w", err)
|
||
}
|
||
|
||
bucketPrefix := "/" + bucket + "/"
|
||
return map[string]interface{}{
|
||
"name": path.Base(info.Key),
|
||
"path": bucketPrefix + info.Key,
|
||
"size": info.Size,
|
||
"size_str": common.FormatBytesInt(info.Size),
|
||
"is_dir": strings.HasSuffix(info.Key, "/"),
|
||
"mod_time": info.LastModified.Format("2006-01-02 15:04:05"),
|
||
}, nil
|
||
}
|
||
|
||
// CreateDir 创建目录
|
||
func (s *Service) CreateDir(connID string, rawPath string) (*filesystem.FileOperationResult, error) {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return nil, fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
if !strings.HasSuffix(key, "/") {
|
||
key += "/"
|
||
}
|
||
|
||
_, err = c.Upload(context.Background(), key, strings.NewReader(""), nil)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("创建目录失败: %w", err)
|
||
}
|
||
|
||
name := path.Base(strings.TrimSuffix(key, "/"))
|
||
return &filesystem.FileOperationResult{
|
||
Path: "/" + bucket + "/" + key,
|
||
Name: name,
|
||
IsDir: true,
|
||
SizeStr: common.FormatBytesInt(0),
|
||
}, nil
|
||
}
|
||
|
||
// CreateFile 创建空文件
|
||
func (s *Service) CreateFile(connID string, rawPath string) (*filesystem.FileOperationResult, error) {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return nil, fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
_, err = c.Upload(context.Background(), key, strings.NewReader(""), nil)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("创建文件失败: %w", err)
|
||
}
|
||
|
||
return &filesystem.FileOperationResult{
|
||
Path: "/" + bucket + "/" + key,
|
||
Name: path.Base(key),
|
||
IsDir: false,
|
||
SizeStr: common.FormatBytesInt(0),
|
||
}, nil
|
||
}
|
||
|
||
// DeletePath 删除文件或目录
|
||
func (s *Service) DeletePath(connID string, rawPath string) (*filesystem.FileOperationResult, error) {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return nil, fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
ctx := context.Background()
|
||
|
||
isDir := strings.HasSuffix(key, "/")
|
||
if !isDir {
|
||
prefix := key + "/"
|
||
listResult, listErr := c.ListFiles(ctx, &oss.ListOptions{Prefix: prefix, MaxKeys: 1})
|
||
if listErr == nil && len(listResult.Files) > 0 {
|
||
isDir = true
|
||
key = prefix
|
||
}
|
||
}
|
||
|
||
infoMap, _ := s.GetFileInfo(connID, "/"+bucket+"/"+key)
|
||
|
||
if isDir {
|
||
prefix := key
|
||
if !strings.HasSuffix(prefix, "/") {
|
||
prefix += "/"
|
||
}
|
||
for {
|
||
listResult, err := c.ListFiles(ctx, &oss.ListOptions{Prefix: prefix, MaxKeys: 1000})
|
||
if err != nil {
|
||
return nil, fmt.Errorf("列举目录文件失败: %w", err)
|
||
}
|
||
if len(listResult.Files) == 0 {
|
||
break
|
||
}
|
||
keys := make([]string, len(listResult.Files))
|
||
for i, f := range listResult.Files {
|
||
keys[i] = f.Key
|
||
}
|
||
if _, err := c.DeleteMultiple(ctx, keys); err != nil {
|
||
return nil, fmt.Errorf("批量删除失败: %w", err)
|
||
}
|
||
if !listResult.IsTruncated {
|
||
break
|
||
}
|
||
}
|
||
c.Delete(ctx, key) // marker 非关键，忽略错误
|
||
} else {
|
||
if err := c.Delete(ctx, key); err != nil {
|
||
return nil, fmt.Errorf("删除失败: %w", err)
|
||
}
|
||
}
|
||
|
||
result := toOssOperationResult(infoMap, isDir)
|
||
result.Deleted = true
|
||
return result, nil
|
||
}
|
||
|
||
// RenamePath 重命名（Copy + Delete）
|
||
func (s *Service) RenamePath(connID string, oldPath string, newPath string) (*filesystem.FileOperationResult, error) {
|
||
oldBucket, oldKey := parseBucketPath(oldPath)
|
||
newBucket, newKey := parseBucketPath(newPath)
|
||
if oldBucket == "" || newBucket == "" {
|
||
return nil, fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
if oldBucket != newBucket {
|
||
return nil, fmt.Errorf("不支持跨桶重命名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, oldBucket, "")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
ctx := context.Background()
|
||
|
||
isDir := strings.HasSuffix(oldKey, "/")
|
||
if !isDir {
|
||
prefix := oldKey + "/"
|
||
listResult, listErr := c.ListFiles(ctx, &oss.ListOptions{Prefix: prefix, MaxKeys: 1})
|
||
if listErr == nil && len(listResult.Files) > 0 {
|
||
isDir = true
|
||
oldKey = prefix
|
||
}
|
||
}
|
||
|
||
if isDir {
|
||
oldPrefix := oldKey
|
||
newPrefix := newKey
|
||
if !strings.HasSuffix(oldPrefix, "/") {
|
||
oldPrefix += "/"
|
||
}
|
||
if !strings.HasSuffix(newPrefix, "/") {
|
||
newPrefix += "/"
|
||
}
|
||
|
||
for {
|
||
listResult, err := c.ListFiles(ctx, &oss.ListOptions{Prefix: oldPrefix, MaxKeys: 1000})
|
||
if err != nil {
|
||
return nil, fmt.Errorf("列举目录文件失败: %w", err)
|
||
}
|
||
if len(listResult.Files) == 0 {
|
||
break
|
||
}
|
||
for _, f := range listResult.Files {
|
||
relativeKey := strings.TrimPrefix(f.Key, oldPrefix)
|
||
if err := c.Copy(ctx, f.Key, newPrefix+relativeKey); err != nil {
|
||
return nil, fmt.Errorf("复制失败: %w", err)
|
||
}
|
||
if err := c.Delete(ctx, f.Key); err != nil {
|
||
return nil, fmt.Errorf("删除源文件失败: %w", err)
|
||
}
|
||
}
|
||
if !listResult.IsTruncated {
|
||
break
|
||
}
|
||
}
|
||
if err := c.Delete(ctx, oldKey); err != nil { // marker
|
||
// 目录标记删除失败不影响重命名结果
|
||
_ = err
|
||
}
|
||
} else {
|
||
if err := c.Copy(ctx, oldKey, newKey); err != nil {
|
||
return nil, fmt.Errorf("复制失败: %w", err)
|
||
}
|
||
if err := c.Delete(ctx, oldKey); err != nil {
|
||
return nil, fmt.Errorf("删除源文件失败: %w", err)
|
||
}
|
||
}
|
||
|
||
infoMap, _ := s.GetFileInfo(connID, newPath)
|
||
result := toOssOperationResult(infoMap, isDir)
|
||
result.OldPath = oldPath
|
||
return result, nil
|
||
}
|
||
|
||
|
||
|
||
// DownloadSiteForPreview 下载 HTML 及其引用的资源到临时目录
|
||
// 对绝对路径（/开头）从 HTML 目录逐级向上嗅探网站根目录
|
||
func (s *Service) DownloadSiteForPreview(connID string, rawPath string) (string, error) {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return "", fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
ctx := context.Background()
|
||
|
||
// 1. 缓存检查：用确定性路径，命中直接返回
|
||
htmlInfo, _ := c.GetFileInfo(ctx, key)
|
||
var htmlSize int64
|
||
var htmlModTime string
|
||
if htmlInfo != nil {
|
||
htmlSize = htmlInfo.Size
|
||
htmlModTime = htmlInfo.LastModified.Format("2006-01-02 15:04:05")
|
||
}
|
||
cacheDir := common.SiteCacheDir("oss", connID, rawPath, htmlSize, htmlModTime)
|
||
|
||
keyDir := path.Dir(key)
|
||
var htmlCachePath string
|
||
if keyDir != "" && keyDir != "." {
|
||
htmlCachePath = filepath.Join(cacheDir, filepath.FromSlash(keyDir), path.Base(key))
|
||
} else {
|
||
htmlCachePath = filepath.Join(cacheDir, path.Base(key))
|
||
}
|
||
if _, err := os.Stat(htmlCachePath); err == nil {
|
||
return htmlCachePath, nil // 缓存命中
|
||
}
|
||
os.RemoveAll(cacheDir)
|
||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||
return "", fmt.Errorf("创建缓存目录失败: %w", err)
|
||
}
|
||
tmpDir := cacheDir
|
||
|
||
htmlLocalPath := htmlCachePath
|
||
if err := os.MkdirAll(filepath.Dir(htmlLocalPath), 0755); err != nil {
|
||
os.RemoveAll(tmpDir)
|
||
return "", fmt.Errorf("创建目录失败: %w", err)
|
||
}
|
||
|
||
// 2. 下载 HTML
|
||
f, err := os.Create(htmlLocalPath)
|
||
if err != nil {
|
||
os.RemoveAll(tmpDir)
|
||
return "", fmt.Errorf("创建临时文件失败: %w", err)
|
||
}
|
||
if err := c.Download(ctx, key, f); err != nil {
|
||
f.Close()
|
||
os.RemoveAll(tmpDir)
|
||
return "", fmt.Errorf("下载 HTML 失败: %w", err)
|
||
}
|
||
f.Close()
|
||
|
||
// 3. 解析 HTML 提取资源路径
|
||
htmlContent, err := os.ReadFile(htmlLocalPath)
|
||
if err != nil {
|
||
return htmlLocalPath, nil // HTML 已下载，资源解析失败不影响
|
||
}
|
||
resources := common.ExtractHtmlResources(string(htmlContent))
|
||
|
||
// 4. 下载资源
|
||
htmlOssDir := keyDir
|
||
if htmlOssDir == "." {
|
||
htmlOssDir = ""
|
||
}
|
||
htmlLocalDir := filepath.Dir(htmlLocalPath)
|
||
|
||
var siteRoot string
|
||
var discoveredDirs []string
|
||
seenDir := make(map[string]bool)
|
||
recordDir := func(ossKey string) {
|
||
dir := path.Dir(ossKey)
|
||
if !seenDir[dir] {
|
||
seenDir[dir] = true
|
||
discoveredDirs = append(discoveredDirs, dir)
|
||
}
|
||
}
|
||
|
||
// 绝对路径资源：串行嗅探 siteRoot，相对路径：并行下载
|
||
type relativeTask struct{ ossKey, localPath string }
|
||
var relativeTasks []relativeTask
|
||
for _, resPath := range resources {
|
||
if common.ShouldSkipResource(resPath) {
|
||
continue
|
||
}
|
||
|
||
isAbsolute := strings.HasPrefix(resPath, "/")
|
||
cleanPath := strings.TrimPrefix(resPath, "/")
|
||
cleanPath = strings.TrimPrefix(cleanPath, "./")
|
||
if cleanPath == "" {
|
||
continue
|
||
}
|
||
|
||
localPath := filepath.Join(htmlLocalDir, filepath.FromSlash(cleanPath))
|
||
|
||
if isAbsolute {
|
||
resolvedKey := resolveAndDownload(c, ctx, htmlOssDir, cleanPath, localPath, &siteRoot)
|
||
if resolvedKey != "" {
|
||
recordDir(resolvedKey)
|
||
}
|
||
} else {
|
||
var ossKey string
|
||
if htmlOssDir != "" {
|
||
ossKey = htmlOssDir + "/" + cleanPath
|
||
} else {
|
||
ossKey = cleanPath
|
||
}
|
||
relativeTasks = append(relativeTasks, relativeTask{ossKey, localPath})
|
||
}
|
||
}
|
||
|
||
// 相对路径资源并行下载（recordDir 需加锁，多 goroutine 并发回调）
|
||
var mu sync.Mutex
|
||
common.RunConcurrent(relativeTasks, common.SiteDownloadConcurrency, func(t relativeTask) {
|
||
if downloadResource(c, ctx, t.ossKey, t.localPath) {
|
||
mu.Lock()
|
||
recordDir(t.ossKey)
|
||
mu.Unlock()
|
||
}
|
||
})
|
||
// 5. 补充下载已发现目录中的剩余文件（覆盖 webpack 动态 chunk 等）
|
||
for _, dir := range discoveredDirs {
|
||
supplementDir(c, ctx, dir, tmpDir, siteRoot)
|
||
}
|
||
|
||
return htmlLocalPath, nil
|
||
}
|
||
|
||
// resolveAndDownload 解析绝对路径并下载：首次嗅探网站根，后续直接使用
|
||
func resolveAndDownload(c oss.OSSProvider, ctx context.Context, htmlOssDir string, cleanPath string, localPath string, siteRoot *string) string {
|
||
if *siteRoot != "" {
|
||
downloadResource(c, ctx, *siteRoot+cleanPath, localPath)
|
||
return *siteRoot + cleanPath
|
||
}
|
||
|
||
// 从 HTML 目录向上逐级探测（同时下载，成功即完成嗅探）
|
||
dir := htmlOssDir
|
||
for {
|
||
var candidate string
|
||
if dir == "" {
|
||
candidate = cleanPath
|
||
} else {
|
||
candidate = dir + "/" + cleanPath
|
||
}
|
||
|
||
if downloadResource(c, ctx, candidate, localPath) {
|
||
if dir == "" {
|
||
*siteRoot = ""
|
||
} else {
|
||
*siteRoot = dir + "/"
|
||
}
|
||
return candidate
|
||
}
|
||
|
||
if dir == "" {
|
||
break
|
||
}
|
||
parent := path.Dir(dir)
|
||
if parent == dir || parent == "." {
|
||
dir = ""
|
||
} else {
|
||
dir = parent
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// downloadResource 下载资源到本地（失败静默，返回是否成功）
|
||
func downloadResource(c oss.OSSProvider, ctx context.Context, ossKey string, localPath string) bool {
|
||
if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
|
||
return false
|
||
}
|
||
f, err := os.Create(localPath)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
if err := c.Download(ctx, ossKey, f); err != nil {
|
||
f.Close()
|
||
os.Remove(localPath)
|
||
return false
|
||
}
|
||
f.Close()
|
||
return true
|
||
}
|
||
|
||
// supplementDir 补充下载远程目录中尚未下载的文件（只处理已知资源所在目录）
|
||
// 数量与单文件大小上限见 common.MaxSupplementFiles / common.MaxSupplementFileSize
|
||
func supplementDir(c oss.OSSProvider, ctx context.Context, remoteDir string, tmpDir string, siteRoot string) {
|
||
prefix := remoteDir + "/"
|
||
var marker string
|
||
count := 0
|
||
for count < common.MaxSupplementFiles {
|
||
result, err := c.ListFiles(ctx, &oss.ListOptions{Prefix: prefix, MaxKeys: 200, Marker: marker})
|
||
if err != nil {
|
||
return
|
||
}
|
||
for _, f := range result.Files {
|
||
if strings.HasSuffix(f.Key, "/") || f.Size == 0 || f.Size > common.MaxSupplementFileSize {
|
||
continue
|
||
}
|
||
if count >= common.MaxSupplementFiles {
|
||
log.Printf("[站点下载] 补充扫描达到上限: count=%d", common.MaxSupplementFiles)
|
||
return
|
||
}
|
||
localPath := filepath.Join(tmpDir, filepath.FromSlash(f.Key))
|
||
if _, err := os.Stat(localPath); err == nil {
|
||
continue
|
||
}
|
||
if downloadResource(c, ctx, f.Key, localPath) {
|
||
count++
|
||
}
|
||
}
|
||
if !result.IsTruncated {
|
||
return
|
||
}
|
||
marker = result.NextMarker
|
||
}
|
||
}
|
||
|
||
// DownloadToTemp 下载文件到本地临时目录（带 SQLite 缓存）
|
||
func (s *Service) DownloadToTemp(connID string, rawPath string) (string, error) {
|
||
// 获取文件元信息用于缓存键
|
||
info, err := s.GetFileInfo(connID, rawPath)
|
||
if err != nil {
|
||
return "", fmt.Errorf("获取文件信息失败: %w", err)
|
||
}
|
||
fileSize, _ := info["size"].(int64)
|
||
modTime, _ := info["mod_time"].(string)
|
||
return storage.DownloadToTempCached("oss", connID, rawPath, fileSize, modTime, func() (string, error) {
|
||
return s.downloadToTempDirect(connID, rawPath)
|
||
})
|
||
}
|
||
|
||
// downloadToTempDirect 实际执行下载（无缓存）
|
||
func (s *Service) downloadToTempDirect(connID string, rawPath string) (string, error) {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return "", fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
f, err := os.CreateTemp("", "udesk-oss-*-"+path.Base(key))
|
||
if err != nil {
|
||
return "", fmt.Errorf("创建临时文件失败: %w", err)
|
||
}
|
||
localPath := f.Name()
|
||
defer f.Close()
|
||
|
||
if err := c.Download(context.Background(), key, f); err != nil {
|
||
os.Remove(localPath)
|
||
return "", fmt.Errorf("下载文件失败: %w", err)
|
||
}
|
||
return localPath, nil
|
||
}
|
||
|
||
// DownloadToTempCached 带缓存的 OSS 下载（命中缓存直接返回本地路径，支持传入文件元信息）
|
||
func (s *Service) DownloadToTempCached(connID, rawPath string, fileSize int64, modTime string) (string, error) {
|
||
return storage.DownloadToTempCached("oss", connID, rawPath, fileSize, modTime, func() (string, error) {
|
||
return s.downloadToTempDirect(connID, rawPath)
|
||
})
|
||
}
|
||
|
||
// GetCommonPaths 返回常用路径
|
||
func (s *Service) GetCommonPaths(connID string) (map[string]string, error) {
|
||
return map[string]string{
|
||
"root": "/",
|
||
}, nil
|
||
}
|
||
|
||
// GetSignedURL 获取预签名 URL
|
||
func (s *Service) GetSignedURL(connID string, rawPath string) (string, error) {
|
||
bucket, key := parseBucketPath(rawPath)
|
||
if bucket == "" {
|
||
return "", fmt.Errorf("路径中缺少桶名")
|
||
}
|
||
|
||
c, err := s.manager.getOrCreateBucketClient(connID, bucket, "")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
|
||
url, err := c.GetSignedURL(context.Background(), key, 1*time.Hour)
|
||
if err != nil {
|
||
return "", fmt.Errorf("获取签名 URL 失败: %w", err)
|
||
}
|
||
return url, nil
|
||
}
|
||
|
||
func toOssOperationResult(m map[string]interface{}, isDir bool) *filesystem.FileOperationResult {
|
||
name, _ := m["name"].(string)
|
||
p, _ := m["path"].(string)
|
||
size, _ := m["size"].(int64)
|
||
modTime, _ := m["mod_time"].(string)
|
||
|
||
return &filesystem.FileOperationResult{
|
||
Path: p,
|
||
Name: name,
|
||
Size: size,
|
||
SizeStr: common.FormatBytesInt(size),
|
||
IsDir: isDir,
|
||
ModTime: modTime,
|
||
}
|
||
}
|