新增: 云OSS存储集成（七牛云+阿里云）+多桶导航+GBK编码自动转换

2026-05-05 03:18:47 +08:00
parent eb5b85e007
commit b4f4b4627d
34 changed files with 5225 additions and 48 deletions
--- a/internal/oss/qiniu/client.go
+++ b/internal/oss/qiniu/client.go
@@ -0,0 +1,570 @@
+package qiniu
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"u-desk/internal/oss"
+)
+
+// Config 七牛云配置
+type Config struct {
+	AccessKey string // 访问密钥
+	SecretKey string // 秘钥
+	Bucket    string // 存储空间名称
+	Region    string // 区域 z0=华东, as0=亚太0区
+	UseHTTPS  bool   // 是否使用 HTTPS
+	UploadDomain string // 上传域名（可选，默认根据 Region 自动选择）
+}
+
+// Client 七牛云客户端
+type Client struct {
+	config     *Config
+	httpClient *http.Client
+	rsAPI      string // 资源管理 API
+	rsfAPI     string // 资源列举 API (RSF)
+	apiAPI     string // API 服务
+}
+
+// NewClient 创建七牛云客户端
+func NewClient(config *Config) (*Client, error) {
+	if config == nil {
+		return nil, oss.NewError("INVALID_CONFIG", "config cannot be nil", nil)
+	}
+	if config.AccessKey == "" || config.SecretKey == "" {
+		return nil, oss.NewError("INVALID_CONFIG", "access key and secret key are required", nil)
+	}
+	if config.Bucket == "" {
+		return nil, oss.NewError("INVALID_CONFIG", "bucket name is required", nil)
+	}
+
+	// 设置默认区域
+	if config.Region == "" {
+		config.Region = "z0" // 华东
+	}
+
+	return &Client{
+		config:     config,
+		httpClient: &http.Client{Timeout: 30 * time.Second},
+		rsAPI:      "http://rs.qiniu.com",
+		rsfAPI:     "http://rsf.qbox.me", // 资源列举 API
+		apiAPI:     "http://api.qiniu.com",
+	}, nil
+}
+
+// generateSignature 生成七牛云管理凭证签名
+// 根据官方文档：https://developer.qiniu.com/kodo/1201/access-token
+func (c *Client) generateSignature(method, path, host, contentType string, body []byte) string {
+	// 七牛云管理凭证签名格式:
+	// signingStr = Method + " " + Path + "\nHost: " + Host + "\n" + [Content-Type] + "\n\n" + [body]
+	var signingStr string
+
+	// 1. Method + " " + Path
+	signingStr = method + " " + path
+
+	// 2. Host header
+	signingStr += "\nHost: " + host
+
+	// 3. Content-Type header (如果设置了)
+	if contentType != "" {
+		signingStr += "\nContent-Type: " + contentType
+	}
+
+	// 4. 两个连续换行符
+	signingStr += "\n\n"
+
+	// 5. Body (如果设置了 Content-Type 且不是 application/octet-stream)
+	if contentType != "" && contentType != "application/octet-stream" && len(body) > 0 {
+		signingStr += string(body)
+	}
+
+	// 使用 HMAC-SHA1 签名
+	h := hmac.New(sha1.New, []byte(c.config.SecretKey))
+	h.Write([]byte(signingStr))
+
+	// Base64 URL 安全编码
+	signature := base64.URLEncoding.EncodeToString(h.Sum(nil))
+
+	return signature
+}
+
+// generateAuthToken 生成管理认证 Token
+func (c *Client) generateAuthToken(method, path, host, contentType string, body []byte) string {
+	signature := c.generateSignature(method, path, host, contentType, body)
+	return "Qiniu " + c.config.AccessKey + ":" + signature
+}
+
+// generateAuthTokenWithQuery 生成管理认证 Token（支持 query string）
+func (c *Client) generateAuthTokenWithQuery(method, path, query, host, contentType string, body []byte) string {
+	// 七牛云管理凭证签名格式:
+	// 如果 query 为非空字符串: signingStr = Method + " " + Path + "?" + query + "\nHost: " + Host + ...
+	// 如果 query 为空: signingStr = Method + " " + Path + "\nHost: " + Host + ...
+	var signingStr string
+
+	// 1. Method + " " + Path
+	signingStr = method + " " + path
+
+	// 2. Query string (如果有)
+	if query != "" {
+		signingStr += "?" + query
+	}
+
+	// 3. Host header
+	signingStr += "\nHost: " + host
+
+	// 4. Content-Type header (如果设置了)
+	if contentType != "" {
+		signingStr += "\nContent-Type: " + contentType
+	}
+
+	// 5. 两个连续换行符
+	signingStr += "\n\n"
+
+	// 6. Body (如果设置了 Content-Type 且不是 application/octet-stream)
+	if contentType != "" && contentType != "application/octet-stream" && len(body) > 0 {
+		signingStr += string(body)
+	}
+
+	// 使用 HMAC-SHA1 签名
+	h := hmac.New(sha1.New, []byte(c.config.SecretKey))
+	h.Write([]byte(signingStr))
+
+	// Base64 URL 安全编码
+	signature := base64.URLEncoding.EncodeToString(h.Sum(nil))
+
+	return "Qiniu " + c.config.AccessKey + ":" + signature
+}
+
+// encodeEntry 编码 EntryURI (bucket:key)
+func (c *Client) encodeEntry(key string) string {
+	entry := c.config.Bucket + ":" + key
+	return base64.URLEncoding.EncodeToString([]byte(entry))
+}
+
+// getUploadDomain 获取上传域名
+func (c *Client) getUploadDomain() string {
+	// 如果配置了自定义上传域名，使用自定义的
+	if c.config.UploadDomain != "" {
+		if c.config.UseHTTPS {
+			return "https://" + c.config.UploadDomain
+		}
+		return "http://" + c.config.UploadDomain
+	}
+
+	// 根据区域选择默认上传域名
+	// 七牛云上传域名格式: up-<region>.qiniup.com 或 upload-<region>.qbox.me
+	scheme := "https://"
+	if !c.config.UseHTTPS {
+		scheme = "http://"
+	}
+
+	// 根据区域返回上传域名
+	switch c.config.Region {
+	case "z0": // 华东
+		return scheme + "up-z0.qiniup.com"
+	case "z1": // 华北
+		return scheme + "up-z1.qiniup.com"
+	case "z2": // 华南
+		return scheme + "up-z2.qiniup.com"
+	case "na0": // 北美
+		return scheme + "up-na0.qiniup.com"
+	case "as0": // 亚太
+		return scheme + "up-as0.qiniup.com"
+	default:
+		// 默认使用华东
+		return scheme + "up-z0.qiniup.com"
+	}
+}
+
+// doRequest 执行 HTTP 请求
+func (c *Client) doRequest(method, path string, body io.Reader) (*http.Response, error) {
+	url := c.rsAPI + path
+
+	// 解析 path 和 query string
+	signPath := path
+	queryString := ""
+	if idx := strings.Index(path, "?"); idx > 0 {
+		signPath = path[:idx]
+		queryString = path[idx+1:] // 去掉问号
+	}
+
+	// 读取 body 用于签名
+	var bodyBytes []byte
+	var err error
+	if body != nil {
+		bodyBytes, err = io.ReadAll(body)
+		if err != nil {
+			return nil, oss.NewError("REQUEST_ERROR", "failed to read request body", err)
+		}
+	}
+
+	req, err := http.NewRequest(method, url, bytes.NewReader(bodyBytes))
+	if err != nil {
+		return nil, oss.NewError("REQUEST_ERROR", "failed to create request", err)
+	}
+
+	// 设置 Content-Type
+	contentType := ""
+	if method == "POST" || method == "PUT" {
+		contentType = "application/x-www-form-urlencoded"
+		req.Header.Set("Content-Type", contentType)
+	}
+
+	// 设置管理认证头（使用新的签名算法，包含 query string）
+	host := "rs.qiniu.com"
+	authToken := c.generateAuthTokenWithQuery(method, signPath, queryString, host, contentType, bodyBytes)
+	req.Header.Set("Authorization", authToken)
+
+	return c.httpClient.Do(req)
+}
+
+// doRSFRequest 执行 RSF (资源列举) API 请求
+// RSF API 使用不同的 host (rsf.qbox.me)
+func (c *Client) doRSFRequest(method, path string) (*http.Response, error) {
+	url := c.rsfAPI + path
+
+	// 解析 path 和 query string
+	signPath := path
+	queryString := ""
+	if idx := strings.Index(path, "?"); idx > 0 {
+		signPath = path[:idx]
+		queryString = path[idx+1:] // 去掉问号
+	}
+
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return nil, oss.NewError("REQUEST_ERROR", "failed to create request", err)
+	}
+
+	// 设置 Content-Type
+	contentType := "application/x-www-form-urlencoded"
+	req.Header.Set("Content-Type", contentType)
+
+	// 设置管理认证头（使用 RSF host）
+	host := "rsf.qbox.me"
+	authToken := c.generateAuthTokenWithQuery(method, signPath, queryString, host, contentType, nil)
+	req.Header.Set("Authorization", authToken)
+
+	return c.httpClient.Do(req)
+}
+
+// Upload 上传文件 (使用表单上传)
+func (c *Client) Upload(ctx context.Context, key string, reader io.Reader, options *oss.UploadOptions) (*oss.UploadResult, error) {
+	// 使用 UploadClient 进行上传
+	uploadClient := NewUploadClient(c.config)
+	return uploadClient.Upload(ctx, key, reader)
+}
+
+// generateUploadToken 生成上传凭证
+func (c *Client) generateUploadToken(key string) string {
+	// 七牛云上传凭证的生成
+	// 1. 创建 putPolicy
+	putPolicy := fmt.Sprintf(`{"scope":"%s:%s","deadline":%d}`,
+		c.config.Bucket, key, time.Now().Add(1*time.Hour).Unix())
+
+	// 2. 对 putPolicy 进行 base64 URL 编码
+	encodedPutPolicy := base64.URLEncoding.EncodeToString([]byte(putPolicy))
+
+	// 3. 对 encodedPutPolicy 进行 HMAC-SHA1 签名
+	h := hmac.New(sha1.New, []byte(c.config.SecretKey))
+	h.Write([]byte(encodedPutPolicy))
+	encodedSign := base64.URLEncoding.EncodeToString(h.Sum(nil))
+
+	// 4. 组合 token
+	return c.config.AccessKey + ":" + encodedSign + ":" + encodedPutPolicy
+}
+
+// generateBucketToken 生成 bucket 级别的上传凭证（用于分片上传 v2）
+func (c *Client) generateBucketToken() string {
+	// 分片上传 v2 需要 bucket 级别的 token
+	// 1. 创建 putPolicy
+	putPolicy := fmt.Sprintf(`{"scope":"%s","deadline":%d}`,
+		c.config.Bucket, time.Now().Add(1*time.Hour).Unix())
+
+	// 2. 对 putPolicy 进行 base64 URL 编码
+	encodedPutPolicy := base64.URLEncoding.EncodeToString([]byte(putPolicy))
+
+	// 3. 对 encodedPutPolicy 进行 HMAC-SHA1 签名
+	h := hmac.New(sha1.New, []byte(c.config.SecretKey))
+	h.Write([]byte(encodedPutPolicy))
+	encodedSign := base64.URLEncoding.EncodeToString(h.Sum(nil))
+
+	// 4. 组合 token
+	return c.config.AccessKey + ":" + encodedSign + ":" + encodedPutPolicy
+}
+
+// resolveDownloadDomain 解析并缓存下载域名
+func (c *Client) resolveDownloadDomain() (string, error) {
+	if c.config.UploadDomain != "" {
+		return c.config.UploadDomain, nil
+	}
+	domains, err := c.GetBucketDomains(context.Background())
+	if err != nil || len(domains) == 0 {
+		return "", fmt.Errorf("无法获取桶 %s 的下载域名: %v", c.config.Bucket, err)
+	}
+	domain := domains[0]
+	if !strings.HasPrefix(domain, "http://") && !strings.HasPrefix(domain, "https://") {
+		domain = "http://" + domain
+	}
+	c.config.UploadDomain = domain
+	return domain, nil
+}
+
+// Download 下载文件
+func (c *Client) Download(ctx context.Context, key string, writer io.Writer) error {
+	baseURL, err := c.resolveDownloadDomain()
+	if err != nil {
+		return oss.NewError("DOWNLOAD_ERROR", err.Error(), err)
+	}
+	url := fmt.Sprintf("%s/%s", baseURL, key)
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return oss.NewError("DOWNLOAD_ERROR", "failed to create request", err)
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return oss.NewError("DOWNLOAD_ERROR", "failed to download file", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return oss.NewError("DOWNLOAD_ERROR", fmt.Sprintf("download failed with status %d", resp.StatusCode), nil)
+	}
+
+	_, err = io.Copy(writer, resp.Body)
+	return err
+}
+
+// Delete 删除文件
+func (c *Client) Delete(ctx context.Context, key string) error {
+	encodedEntry := c.encodeEntry(key)
+	path := "/delete/" + encodedEntry
+
+	resp, err := c.doRequest("POST", path, nil)
+	if err != nil {
+		return oss.NewError("DELETE_ERROR", "failed to delete file", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 200 || resp.StatusCode == 612 {
+		return nil
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	return oss.NewError("DELETE_ERROR", fmt.Sprintf("delete failed with status %d: %s", resp.StatusCode, string(body)), nil)
+}
+
+// DeleteMultiple 批量删除文件
+func (c *Client) DeleteMultiple(ctx context.Context, keys []string) (*oss.DeleteResult, error) {
+	result := &oss.DeleteResult{
+		Deleted: make([]string, 0),
+		Errors:  make([]string, 0),
+	}
+
+	for _, key := range keys {
+		if err := c.Delete(ctx, key); err != nil {
+			result.Errors = append(result.Errors, key)
+		} else {
+			result.Deleted = append(result.Deleted, key)
+		}
+	}
+
+	return result, nil
+}
+
+// GetFileInfo 获取文件信息
+func (c *Client) GetFileInfo(ctx context.Context, key string) (*oss.FileInfo, error) {
+	encodedEntry := c.encodeEntry(key)
+	path := "/stat/" + encodedEntry
+
+	resp, err := c.doRequest("GET", path, nil)
+	if err != nil {
+		return nil, oss.NewError("STAT_ERROR", "failed to get file info", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, oss.NewError("STAT_ERROR", "failed to read response", err)
+	}
+
+	if resp.StatusCode == 612 {
+		return nil, oss.ErrFileNotFound
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, oss.NewError("STAT_ERROR", fmt.Sprintf("stat failed with status %d: %s", resp.StatusCode, string(body)), nil)
+	}
+
+	// 解析响应 (简化实现)
+	// 实际响应格式: {"hash":"xxx","fsize":123,"mimeType":"xxx","putTime":123}
+	// 这里返回一个简化的 FileInfo
+	return &oss.FileInfo{
+		Key: key,
+	}, nil
+}
+
+// ListFiles 列举文件
+func (c *Client) ListFiles(ctx context.Context, options *oss.ListOptions) (*oss.ListResult, error) {
+	if options == nil {
+		options = &oss.ListOptions{}
+	}
+
+	if options.MaxKeys == 0 {
+		options.MaxKeys = 100
+	}
+
+	// 构建查询参数
+	path := fmt.Sprintf("/list?bucket=%s&limit=%d", c.config.Bucket, options.MaxKeys)
+	if options.Prefix != "" {
+		path += "&prefix=" + options.Prefix
+	}
+	if options.Marker != "" {
+		path += "&marker=" + options.Marker
+	}
+
+	// 使用 GET 方法和 RSF API
+	resp, err := c.doRSFRequest("GET", path)
+	if err != nil {
+		return nil, oss.NewError("LIST_ERROR", "failed to list files", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, oss.NewError("LIST_ERROR", "failed to read response", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, oss.NewError("LIST_ERROR", fmt.Sprintf("list failed with status %d: %s", resp.StatusCode, string(body)), nil)
+	}
+
+	// 解析响应
+	// 响应格式: {"marker":"","commonPrefixes":[],"items":[{"key":"xxx","hash":"xxx","fsize":123,...}]}
+	var listResp struct {
+		Marker        string `json:"marker"`
+		CommonPrefixes []string `json:"commonPrefixes"`
+		Items         []struct {
+			Key      string `json:"key"`
+			Hash     string `json:"hash"`
+			Fsize    int64  `json:"fsize"`
+			MimeType string `json:"mimeType"`
+			PutTime  int64  `json:"putTime"`
+		} `json:"items"`
+	}
+
+	if err := json.Unmarshal(body, &listResp); err != nil {
+		return nil, oss.NewError("LIST_ERROR", "failed to parse response", err)
+	}
+
+	// 转换为统一格式
+	files := make([]oss.FileInfo, 0, len(listResp.Items))
+	for _, item := range listResp.Items {
+		files = append(files, oss.FileInfo{
+			Key:         item.Key,
+			Size:        item.Fsize,
+			ETag:        item.Hash,
+			ContentType: item.MimeType,
+		})
+	}
+
+	return &oss.ListResult{
+		Files:       files,
+		IsTruncated: listResp.Marker != "",
+		NextMarker:  listResp.Marker,
+		Prefixes:    listResp.CommonPrefixes,
+	}, nil
+}
+
+// GetSignedURL 获取预签名URL
+func (c *Client) GetSignedURL(ctx context.Context, key string, expiresIn time.Duration) (string, error) {
+	// 七牛云私有空间下载需要生成私有下载 URL
+	deadline := time.Now().Add(expiresIn).Unix()
+
+	// 构建 download URL
+	baseURL, err := c.resolveDownloadDomain()
+	if err != nil {
+		return "", err
+	}
+	downloadURL := fmt.Sprintf("%s/%s", baseURL, key)
+
+	// 生成签名
+	h := hmac.New(sha1.New, []byte(c.config.SecretKey))
+	signStr := fmt.Sprintf("%s\n%d", downloadURL, deadline)
+	h.Write([]byte(signStr))
+	sign := base64.URLEncoding.EncodeToString(h.Sum(nil))
+
+	// 构建最终 URL
+	signedURL := fmt.Sprintf("%s?e=%d&token=%s:%s", downloadURL, deadline, c.config.AccessKey, sign)
+
+	return signedURL, nil
+}
+
+// Copy 复制文件
+func (c *Client) Copy(ctx context.Context, sourceKey, targetKey string) error {
+	sourceEntry := c.encodeEntry(sourceKey)
+	targetEntry := c.encodeEntry(targetKey)
+	path := "/copy/" + sourceEntry + "/" + targetEntry
+
+	resp, err := c.doRequest("POST", path, nil)
+	if err != nil {
+		return oss.NewError("COPY_ERROR", "failed to copy file", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 200 {
+		return nil
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	return oss.NewError("COPY_ERROR", fmt.Sprintf("copy failed with status %d: %s", resp.StatusCode, string(body)), nil)
+}
+
+// Move 移动/重命名文件
+func (c *Client) Move(ctx context.Context, sourceKey, targetKey string) error {
+	sourceEntry := c.encodeEntry(sourceKey)
+	targetEntry := c.encodeEntry(targetKey)
+	path := "/move/" + sourceEntry + "/" + targetEntry
+
+	resp, err := c.doRequest("POST", path, nil)
+	if err != nil {
+		return oss.NewError("MOVE_ERROR", "failed to move file", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 200 {
+		return nil
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+	return oss.NewError("MOVE_ERROR", fmt.Sprintf("move failed with status %d: %s", resp.StatusCode, string(body)), nil)
+}
+
+// Exists 检查文件是否存在
+func (c *Client) Exists(ctx context.Context, key string) (bool, error) {
+	_, err := c.GetFileInfo(ctx, key)
+	if err == oss.ErrFileNotFound {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// Close 关闭连接
+func (c *Client) Close() error {
+	c.httpClient.CloseIdleConnections()
+	return nil
+}