feat: 添加同步进度显示和中文界面支持

refactor: 重构同步引擎以支持进度跟踪
style: 更新前端界面为中文
docs: 更新README为中文文档
This commit is contained in:
panw
2026-04-01 10:43:51 +08:00
parent 34944518f0
commit 5eff309a9f
13 changed files with 469 additions and 154 deletions

View File

@@ -2,12 +2,13 @@ package database
import (
"os"
"path/filepath"
"testing"
"gitm/internal/models"
)
func TestDatabase(t *testing.T) {
tmpDB := "/tmp/test_gitm.db"
tmpDB := filepath.Join(os.TempDir(), "test_gitm.db")
defer os.Remove(tmpDB)
if err := Initialize(tmpDB); err != nil {

View File

@@ -30,11 +30,26 @@ func NewClient(serverURL, token string) (*Client, error) {
}
func (c *Client) do(path string) (*http.Response, error) {
apiURL := c.baseURL.JoinPath(path)
q := apiURL.Query()
reqURL := c.baseURL.JoinPath(path)
q := reqURL.Query()
q.Set("token", c.token)
apiURL.RawQuery = q.Encode()
resp, err := c.httpClient.Get(apiURL.String())
reqURL.RawQuery = q.Encode()
resp, err := c.httpClient.Get(reqURL.String())
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
return resp, nil
}
func (c *Client) doSearch(path string, params map[string]string) (*http.Response, error) {
reqURL := c.baseURL.JoinPath(path)
q := reqURL.Query()
q.Set("token", c.token)
for k, v := range params {
q.Set(k, v)
}
reqURL.RawQuery = q.Encode()
resp, err := c.httpClient.Get(reqURL.String())
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
@@ -59,8 +74,10 @@ func (c *Client) ValidateToken() (*GiteaUser, error) {
}
func (c *Client) SearchRepos(page, limit int) ([]GiteaRepo, error) {
path := fmt.Sprintf("/api/v1/repos/search?page=%d&limit=%d", page, limit)
resp, err := c.do(path)
resp, err := c.doSearch("/api/v1/repos/search", map[string]string{
"page": fmt.Sprintf("%d", page),
"limit": fmt.Sprintf("%d", limit),
})
if err != nil {
return nil, err
}

View File

@@ -229,10 +229,7 @@ func HandleGetSyncStatus(engine *sync.Engine) gin.HandlerFunc {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid server ID"})
return
}
status := "idle"
if engine.IsSyncing(id) {
status = "syncing"
}
c.JSON(http.StatusOK, gin.H{"server_id": id, "status": status})
progress := engine.GetProgress(id)
c.JSON(http.StatusOK, progress)
}
}

View File

@@ -2,6 +2,7 @@ package sync
import (
"fmt"
"net/url"
"os"
"os/exec"
"path/filepath"
@@ -13,16 +14,75 @@ import (
"gitm/internal/models"
)
type RepoProgress struct {
FullName string `json:"full_name"`
Action string `json:"action"` // "cloning" or "fetching"
}
type ServerProgress struct {
ServerID int64 `json:"server_id"`
Total int `json:"total"`
Completed int `json:"completed"`
SuccessCount int `json:"success_count"`
FailedCount int `json:"failed_count"`
Current *RepoProgress `json:"current"`
Status string `json:"status"` // "syncing" or "idle"
}
type Engine struct {
maxConcurrent int
mu sync.Mutex
activeTasks map[int64]string
progress map[int64]*ServerProgress
progressMu sync.RWMutex
}
func NewEngine(maxConcurrent int) *Engine {
return &Engine{
maxConcurrent: maxConcurrent,
activeTasks: make(map[int64]string),
progress: make(map[int64]*ServerProgress),
}
}
func (e *Engine) GetProgress(serverID int64) *ServerProgress {
e.progressMu.RLock()
defer e.progressMu.RUnlock()
if p, ok := e.progress[serverID]; ok {
return p
}
return &ServerProgress{ServerID: serverID, Status: "idle"}
}
func (e *Engine) setProgress(serverID int64, p *ServerProgress) {
e.progressMu.Lock()
defer e.progressMu.Unlock()
e.progress[serverID] = p
}
func (e *Engine) updateProgressCounts(serverID int64, successDelta, failedDelta int) {
e.progressMu.Lock()
defer e.progressMu.Unlock()
if p, ok := e.progress[serverID]; ok {
p.SuccessCount += successDelta
p.FailedCount += failedDelta
p.Completed = p.SuccessCount + p.FailedCount
}
}
func (e *Engine) setCurrentRepo(serverID int64, fullName, action string) {
e.progressMu.Lock()
defer e.progressMu.Unlock()
if p, ok := e.progress[serverID]; ok {
p.Current = &RepoProgress{FullName: fullName, Action: action}
}
}
func (e *Engine) clearCurrentRepo(serverID int64) {
e.progressMu.Lock()
defer e.progressMu.Unlock()
if p, ok := e.progress[serverID]; ok {
p.Current = nil
}
}
@@ -47,33 +107,52 @@ func (e *Engine) SyncServer(serverID int64) error {
return fmt.Errorf("failed to get server: %w", err)
}
// Initialize progress
e.setProgress(serverID, &ServerProgress{
ServerID: serverID,
Total: 0,
Status: "syncing",
})
syncLog := &models.SyncLog{
ServerID: serverID,
Status: "in_progress",
Message: "Starting sync",
Message: "开始同步",
StartedAt: time.Now(),
}
if err := database.CreateSyncLog(syncLog); err != nil {
return fmt.Errorf("failed to create sync log: %w", err)
}
e.setCurrentRepo(serverID, "", "验证认证中")
client, err := gitea.NewClient(server.URL, server.Token)
if err != nil {
e.finishLog(syncLog, "failed", fmt.Sprintf("Failed to create client: %v", err))
e.finishLog(syncLog, "failed", fmt.Sprintf("创建客户端失败: %v", err))
e.setProgress(serverID, &ServerProgress{ServerID: serverID, Status: "idle"})
return err
}
if _, err := client.ValidateToken(); err != nil {
e.finishLog(syncLog, "failed", fmt.Sprintf("Authentication failed: %v", err))
e.finishLog(syncLog, "failed", fmt.Sprintf("认证失败: %v", err))
e.setProgress(serverID, &ServerProgress{ServerID: serverID, Status: "idle"})
return err
}
e.setCurrentRepo(serverID, "", "获取仓库列表中")
giteaRepos, err := client.GetAllRepos()
if err != nil {
e.finishLog(syncLog, "failed", fmt.Sprintf("Failed to fetch repos: %v", err))
e.finishLog(syncLog, "failed", fmt.Sprintf("获取仓库列表失败: %v", err))
e.setProgress(serverID, &ServerProgress{ServerID: serverID, Status: "idle"})
return err
}
// Update total count
e.progressMu.Lock()
if p, ok := e.progress[serverID]; ok {
p.Total = len(giteaRepos)
}
e.progressMu.Unlock()
reposDir, _ := database.GetSetting("repos_dir")
if reposDir == "" {
reposDir = "./data/repos"
@@ -81,11 +160,8 @@ func (e *Engine) SyncServer(serverID int64) error {
serverDir := filepath.Join(reposDir, fmt.Sprintf("server_%d_%s", serverID, server.Name))
os.MkdirAll(serverDir, 0755)
successCount := 0
failedCount := 0
sem := make(chan struct{}, e.maxConcurrent)
var wg sync.WaitGroup
var resultsMu sync.Mutex
for _, gr := range giteaRepos {
wg.Add(1)
@@ -99,7 +175,8 @@ func (e *Engine) SyncServer(serverID int64) error {
var err error
if existingRepo == nil {
err = e.cloneMirror(giteaRepo.CloneURL, localPath)
e.setCurrentRepo(serverID, giteaRepo.FullName, "克隆")
err = e.cloneMirror(giteaRepo.CloneURL, server.Token, localPath)
if err == nil {
repo := &models.Repo{
ServerID: serverID,
@@ -112,15 +189,25 @@ func (e *Engine) SyncServer(serverID int64) error {
database.CreateRepo(repo)
}
} else {
err = e.fetchMirror(localPath)
if err == nil {
database.UpdateRepoSyncStatus(existingRepo.ID, "success")
if _, statErr := os.Stat(localPath); statErr != nil {
e.setCurrentRepo(serverID, giteaRepo.FullName, "克隆")
err = e.cloneMirror(giteaRepo.CloneURL, server.Token, localPath)
if err == nil {
existingRepo.LocalPath = localPath
database.UpdateRepoSyncStatus(existingRepo.ID, "success")
}
} else {
e.setCurrentRepo(serverID, giteaRepo.FullName, "同步")
err = e.fetchMirror(localPath)
if err == nil {
database.UpdateRepoSyncStatus(existingRepo.ID, "success")
}
}
}
resultsMu.Lock()
e.clearCurrentRepo(serverID)
if err != nil {
failedCount++
e.updateProgressCounts(serverID, 0, 1)
failLog := &models.SyncLog{
ServerID: serverID,
RepoID: getRepoID(serverID, giteaRepo.FullName),
@@ -130,26 +217,48 @@ func (e *Engine) SyncServer(serverID int64) error {
}
database.CreateSyncLog(failLog)
} else {
successCount++
e.updateProgressCounts(serverID, 1, 0)
}
resultsMu.Unlock()
}(gr)
}
wg.Wait()
database.UpdateServerLastSync(serverID)
message := fmt.Sprintf("Sync completed: %d succeeded, %d failed", successCount, failedCount)
// Get final counts for message
p := e.GetProgress(serverID)
message := fmt.Sprintf("同步完成: %d 成功, %d 失败", p.SuccessCount, p.FailedCount)
e.finishLog(syncLog, "success", message)
// Mark as idle but keep final progress visible briefly
e.progressMu.Lock()
finalP := e.progress[serverID]
if finalP != nil {
finalP.Current = nil
finalP.Status = "completed"
}
e.progressMu.Unlock()
// Clear after 30 seconds
go func() {
time.Sleep(30 * time.Second)
e.progressMu.Lock()
if p, ok := e.progress[serverID]; ok && p.Status == "completed" {
p.Status = "idle"
}
e.progressMu.Unlock()
}()
return nil
}
func (e *Engine) cloneMirror(url, path string) error {
func (e *Engine) cloneMirror(cloneURL, token, path string) error {
if _, err := os.Stat(path); err == nil {
return nil
}
os.MkdirAll(filepath.Dir(path), 0755)
cmd := exec.Command("git", "clone", "--mirror", url, path)
authURL := injectToken(cloneURL, token)
cmd := exec.Command("git", "clone", "--mirror", authURL, path)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("clone failed: %w - %s", err, string(output))
@@ -157,6 +266,15 @@ func (e *Engine) cloneMirror(url, path string) error {
return nil
}
func injectToken(rawURL, token string) string {
u, err := url.Parse(rawURL)
if err != nil {
return rawURL
}
u.User = url.UserPassword(token, "x-oauth-basic")
return u.String()
}
func (e *Engine) fetchMirror(path string) error {
cmd := exec.Command("git", "--git-dir", path, "fetch", "--all", "--prune")
output, err := cmd.CombinedOutput()