From 5eff309a9f55f75b4241f28b9fbbcc33a9ef87ba Mon Sep 17 00:00:00 2001 From: panw Date: Wed, 1 Apr 2026 10:43:51 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E8=BF=9B=E5=BA=A6=E6=98=BE=E7=A4=BA=E5=92=8C=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E7=95=8C=E9=9D=A2=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit refactor: 重构同步引擎以支持进度跟踪 style: 更新前端界面为中文 docs: 更新README为中文文档 --- README.md | 181 ++++++++++++++++++++++------- internal/database/database_test.go | 3 +- internal/gitea/client.go | 29 ++++- internal/handler/server.go | 7 +- internal/sync/engine.go | 154 +++++++++++++++++++++--- main.go | 25 ++-- web/src/views/Dashboard.vue | 29 ++--- web/src/views/Layout.vue | 14 +-- web/src/views/Login.vue | 12 +- web/src/views/Logs.vue | 14 ++- web/src/views/Repos.vue | 15 +-- web/src/views/Servers.vue | 122 ++++++++++++++----- web/src/views/Settings.vue | 18 +-- 13 files changed, 469 insertions(+), 154 deletions(-) diff --git a/README.md b/README.md index ab13dc2..2980085 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,161 @@ -# GitM - Gitea Repository Sync Tool +# GitM - Gitea 仓库镜像同步工具 -Cross-platform tool to synchronize all repositories from multiple Gitea servers to local storage. +将多个 Gitea 服务器的仓库自动镜像同步到本地存储，支持定时同步和手动同步。 -## Features +## 功能特性 -- Single binary deployment (frontend embedded) -- Web UI for management (Vue 3 + Element Plus) -- SQLite database -- JWT authentication -- Scheduled and manual sync -- Cross-platform (Windows, Linux) +- **单文件部署** - 前端内嵌到 Go 二进制，无需额外依赖 +- **Web 管理界面** - Vue 3 + Element Plus，中文界面 +- **多服务器管理** - 同时管理多个 Gitea 服务器 +- **仓库自动发现** - 一键发现服务器上的所有仓库 +- **镜像同步** - 使用 `git clone --mirror` 完整镜像，支持 Token 认证 +- **实时进度** - 同步过程实时展示当前克隆/同步的仓库及进度 +- **定时同步** - 可配置每个服务器的自动同步间隔 +- **SQLite 存储** - 轻量级数据库，无需额外安装 -## Build +## 快速开始 + +### 前置要求 + +- Go 1.21+（需要 CGO 支持 SQLite） +- GCC（Windows 下需安装 MinGW-w64） +- Node.js 18+（仅开发时需要） + +### 安装与运行 ```bash -# Build binary with embedded frontend -go build -o bin/gitm . +# 1. 初始化（设置管理员密码） +$env:CGO_ENABLED="1"; go run main.go --init -# Or with CGO for SQLite support -CGO_ENABLED=1 go build -o bin/gitm . +# 2. 启动服务（默认监听 :9000） +$env:CGO_ENABLED="1"; go run main.go ``` -## Run +启动后浏览器访问 `http://localhost:9000` + +### 使用流程 + +1. **登录** - 使用初始化时设置的密码登录 +2. **添加服务器** - 填写 Gitea 服务器地址和 API Token +3. **测试连接** - 验证服务器和 Token 是否可用 +4. **发现仓库** - 扫描服务器上的所有仓库 +5. **同步** - 将仓库镜像克隆到本地，支持手动和定时同步 + +## 打包部署 ```bash -# First-time initialization (set admin password) -./bin/gitm --init +# 构建前端 +cd web +npm install +npm run build +cd .. -# Start server (default :9000) -./bin/gitm - -# Custom address -./bin/gitm --addr :9090 - -# Custom data directory -./bin/gitm --data /path/to/data +# 打包为单文件 exe +$env:CGO_ENABLED="1"; go build -o gitm.exe ``` -## Usage +打包后的 `gitm.exe` 包含前后端，直接运行即可。 -1. Initialize: `./bin/gitm --init` -- sets admin password -2. Start: `./bin/gitm` -- starts web server -3. Open browser to `http://localhost:9000` -4. Login and add Gitea servers -5. Discover repos and sync - -## Development +### 运行 ```bash -# Frontend dev server -cd web && npm install && npm run dev +# 首次初始化 +.\gitm.exe --init -# Backend -go run main.go +# 启动服务 +.\gitm.exe -# Run tests -go test ./internal/gitea/... -v -CGO_ENABLED=1 go test ./internal/database/... -v +# 指定监听地址 +.\gitm.exe --addr :8080 + +# 指定数据目录 +.\gitm.exe --data /path/to/data ``` + +## 开发模式 + +前后端分别启动，支持热更新： + +```bash +# 终端 1：启动后端（默认 :9000） +$env:CGO_ENABLED="1"; go run main.go + +# 终端 2：启动前端开发服务器（:5173，自动代理 API 到 :9000） +cd web +npm install +npm run dev +``` + +开发时访问 `http://localhost:5173` + +## 命令行参数 + +| 参数 | 说明 | 默认值 | +|------|------|--------| +| `--init` | 初始化数据库并设置管理员密码 | - | +| `--addr` | 监听地址 | `:9000` | +| `--data` | 数据存储目录 | `./data` | + +## 系统设置 + +通过 Web 界面可配置： + +- **管理员密码** - 修改登录密码 +- **监听地址** - 修改服务端口（需重启） +- **仓库目录** - 仓库镜像存储路径 +- **最大并发数** - 同步时的并发数量（1-10） + +## 项目结构 + +``` +├── main.go # 程序入口 +├── internal/ +│ ├── config/ # 配置管理 +│ ├── database/ # SQLite 数据库操作 +│ ├── gitea/ # Gitea API 客户端 +│ ├── handler/ # HTTP 请求处理 +│ ├── middleware/ # JWT 认证中间件 +│ ├── models/ # 数据模型 +│ └── sync/ # 同步引擎与调度器 +├── web/ # Vue 3 前端 +│ ├── src/ +│ │ ├── api/ # API 请求封装 +│ │ ├── views/ # 页面组件 +│ │ ├── stores/ # Pinia 状态管理 +│ │ └── router/ # 路由配置 +│ └── vite.config.ts +└── data/ # 运行时数据（自动创建） + ├── gitm.db # SQLite 数据库 + └── repos/ # 镜像仓库存储 +``` + +## API 接口 + +| 方法 | 路径 | 说明 | +|------|------|------| +| POST | `/api/login` | 登录 | +| POST | `/api/init` | 初始化密码 | +| GET | `/api/settings` | 获取系统设置 | +| PUT | `/api/settings` | 更新系统设置 | +| GET | `/api/servers` | 服务器列表 | +| POST | `/api/servers` | 添加服务器 | +| PUT | `/api/servers/:id` | 更新服务器 | +| DELETE | `/api/servers/:id` | 删除服务器 | +| POST | `/api/servers/test` | 测试连接 | +| GET | `/api/servers/:id/repos` | 仓库列表 | +| POST | `/api/servers/:id/discover` | 发现仓库 | +| POST | `/api/servers/:id/sync` | 同步服务器 | +| GET | `/api/servers/:id/sync/status` | 同步进度 | +| POST | `/api/sync/all` | 同步全部服务器 | +| GET | `/api/sync/logs` | 同步日志 | +| GET | `/api/sync/stats` | 同步统计 | + +## 技术栈 + +**后端：** Go / Gin / SQLite / JWT + +**前端：** Vue 3 / TypeScript / Element Plus / Vite / Pinia + +## License + +MIT diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 553a975..3952557 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -2,12 +2,13 @@ package database import ( "os" + "path/filepath" "testing" "gitm/internal/models" ) func TestDatabase(t *testing.T) { - tmpDB := "/tmp/test_gitm.db" + tmpDB := filepath.Join(os.TempDir(), "test_gitm.db") defer os.Remove(tmpDB) if err := Initialize(tmpDB); err != nil { diff --git a/internal/gitea/client.go b/internal/gitea/client.go index bfa6822..32e705e 100644 --- a/internal/gitea/client.go +++ b/internal/gitea/client.go @@ -30,11 +30,26 @@ func NewClient(serverURL, token string) (*Client, error) { } func (c *Client) do(path string) (*http.Response, error) { - apiURL := c.baseURL.JoinPath(path) - q := apiURL.Query() + reqURL := c.baseURL.JoinPath(path) + q := reqURL.Query() q.Set("token", c.token) - apiURL.RawQuery = q.Encode() - resp, err := c.httpClient.Get(apiURL.String()) + reqURL.RawQuery = q.Encode() + resp, err := c.httpClient.Get(reqURL.String()) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + return resp, nil +} + +func (c *Client) doSearch(path string, params map[string]string) (*http.Response, error) { + reqURL := c.baseURL.JoinPath(path) + q := reqURL.Query() + q.Set("token", c.token) + for k, v := range params { + q.Set(k, v) + } + reqURL.RawQuery = q.Encode() + resp, err := c.httpClient.Get(reqURL.String()) if err != nil { return nil, fmt.Errorf("request failed: %w", err) } @@ -59,8 +74,10 @@ func (c *Client) ValidateToken() (*GiteaUser, error) { } func (c *Client) SearchRepos(page, limit int) ([]GiteaRepo, error) { - path := fmt.Sprintf("/api/v1/repos/search?page=%d&limit=%d", page, limit) - resp, err := c.do(path) + resp, err := c.doSearch("/api/v1/repos/search", map[string]string{ + "page": fmt.Sprintf("%d", page), + "limit": fmt.Sprintf("%d", limit), + }) if err != nil { return nil, err } diff --git a/internal/handler/server.go b/internal/handler/server.go index a39460f..d3ee419 100644 --- a/internal/handler/server.go +++ b/internal/handler/server.go @@ -229,10 +229,7 @@ func HandleGetSyncStatus(engine *sync.Engine) gin.HandlerFunc { c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid server ID"}) return } - status := "idle" - if engine.IsSyncing(id) { - status = "syncing" - } - c.JSON(http.StatusOK, gin.H{"server_id": id, "status": status}) + progress := engine.GetProgress(id) + c.JSON(http.StatusOK, progress) } } diff --git a/internal/sync/engine.go b/internal/sync/engine.go index 2dd1831..4fa7318 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -2,6 +2,7 @@ package sync import ( "fmt" + "net/url" "os" "os/exec" "path/filepath" @@ -13,16 +14,75 @@ import ( "gitm/internal/models" ) +type RepoProgress struct { + FullName string `json:"full_name"` + Action string `json:"action"` // "cloning" or "fetching" +} + +type ServerProgress struct { + ServerID int64 `json:"server_id"` + Total int `json:"total"` + Completed int `json:"completed"` + SuccessCount int `json:"success_count"` + FailedCount int `json:"failed_count"` + Current *RepoProgress `json:"current"` + Status string `json:"status"` // "syncing" or "idle" +} + type Engine struct { maxConcurrent int mu sync.Mutex activeTasks map[int64]string + progress map[int64]*ServerProgress + progressMu sync.RWMutex } func NewEngine(maxConcurrent int) *Engine { return &Engine{ maxConcurrent: maxConcurrent, activeTasks: make(map[int64]string), + progress: make(map[int64]*ServerProgress), + } +} + +func (e *Engine) GetProgress(serverID int64) *ServerProgress { + e.progressMu.RLock() + defer e.progressMu.RUnlock() + if p, ok := e.progress[serverID]; ok { + return p + } + return &ServerProgress{ServerID: serverID, Status: "idle"} +} + +func (e *Engine) setProgress(serverID int64, p *ServerProgress) { + e.progressMu.Lock() + defer e.progressMu.Unlock() + e.progress[serverID] = p +} + +func (e *Engine) updateProgressCounts(serverID int64, successDelta, failedDelta int) { + e.progressMu.Lock() + defer e.progressMu.Unlock() + if p, ok := e.progress[serverID]; ok { + p.SuccessCount += successDelta + p.FailedCount += failedDelta + p.Completed = p.SuccessCount + p.FailedCount + } +} + +func (e *Engine) setCurrentRepo(serverID int64, fullName, action string) { + e.progressMu.Lock() + defer e.progressMu.Unlock() + if p, ok := e.progress[serverID]; ok { + p.Current = &RepoProgress{FullName: fullName, Action: action} + } +} + +func (e *Engine) clearCurrentRepo(serverID int64) { + e.progressMu.Lock() + defer e.progressMu.Unlock() + if p, ok := e.progress[serverID]; ok { + p.Current = nil } } @@ -47,33 +107,52 @@ func (e *Engine) SyncServer(serverID int64) error { return fmt.Errorf("failed to get server: %w", err) } + // Initialize progress + e.setProgress(serverID, &ServerProgress{ + ServerID: serverID, + Total: 0, + Status: "syncing", + }) + syncLog := &models.SyncLog{ ServerID: serverID, Status: "in_progress", - Message: "Starting sync", + Message: "开始同步", StartedAt: time.Now(), } if err := database.CreateSyncLog(syncLog); err != nil { return fmt.Errorf("failed to create sync log: %w", err) } + e.setCurrentRepo(serverID, "", "验证认证中") client, err := gitea.NewClient(server.URL, server.Token) if err != nil { - e.finishLog(syncLog, "failed", fmt.Sprintf("Failed to create client: %v", err)) + e.finishLog(syncLog, "failed", fmt.Sprintf("创建客户端失败: %v", err)) + e.setProgress(serverID, &ServerProgress{ServerID: serverID, Status: "idle"}) return err } if _, err := client.ValidateToken(); err != nil { - e.finishLog(syncLog, "failed", fmt.Sprintf("Authentication failed: %v", err)) + e.finishLog(syncLog, "failed", fmt.Sprintf("认证失败: %v", err)) + e.setProgress(serverID, &ServerProgress{ServerID: serverID, Status: "idle"}) return err } + e.setCurrentRepo(serverID, "", "获取仓库列表中") giteaRepos, err := client.GetAllRepos() if err != nil { - e.finishLog(syncLog, "failed", fmt.Sprintf("Failed to fetch repos: %v", err)) + e.finishLog(syncLog, "failed", fmt.Sprintf("获取仓库列表失败: %v", err)) + e.setProgress(serverID, &ServerProgress{ServerID: serverID, Status: "idle"}) return err } + // Update total count + e.progressMu.Lock() + if p, ok := e.progress[serverID]; ok { + p.Total = len(giteaRepos) + } + e.progressMu.Unlock() + reposDir, _ := database.GetSetting("repos_dir") if reposDir == "" { reposDir = "./data/repos" @@ -81,11 +160,8 @@ func (e *Engine) SyncServer(serverID int64) error { serverDir := filepath.Join(reposDir, fmt.Sprintf("server_%d_%s", serverID, server.Name)) os.MkdirAll(serverDir, 0755) - successCount := 0 - failedCount := 0 sem := make(chan struct{}, e.maxConcurrent) var wg sync.WaitGroup - var resultsMu sync.Mutex for _, gr := range giteaRepos { wg.Add(1) @@ -99,7 +175,8 @@ func (e *Engine) SyncServer(serverID int64) error { var err error if existingRepo == nil { - err = e.cloneMirror(giteaRepo.CloneURL, localPath) + e.setCurrentRepo(serverID, giteaRepo.FullName, "克隆") + err = e.cloneMirror(giteaRepo.CloneURL, server.Token, localPath) if err == nil { repo := &models.Repo{ ServerID: serverID, @@ -112,15 +189,25 @@ func (e *Engine) SyncServer(serverID int64) error { database.CreateRepo(repo) } } else { - err = e.fetchMirror(localPath) - if err == nil { - database.UpdateRepoSyncStatus(existingRepo.ID, "success") + if _, statErr := os.Stat(localPath); statErr != nil { + e.setCurrentRepo(serverID, giteaRepo.FullName, "克隆") + err = e.cloneMirror(giteaRepo.CloneURL, server.Token, localPath) + if err == nil { + existingRepo.LocalPath = localPath + database.UpdateRepoSyncStatus(existingRepo.ID, "success") + } + } else { + e.setCurrentRepo(serverID, giteaRepo.FullName, "同步") + err = e.fetchMirror(localPath) + if err == nil { + database.UpdateRepoSyncStatus(existingRepo.ID, "success") + } } } - resultsMu.Lock() + e.clearCurrentRepo(serverID) if err != nil { - failedCount++ + e.updateProgressCounts(serverID, 0, 1) failLog := &models.SyncLog{ ServerID: serverID, RepoID: getRepoID(serverID, giteaRepo.FullName), @@ -130,26 +217,48 @@ func (e *Engine) SyncServer(serverID int64) error { } database.CreateSyncLog(failLog) } else { - successCount++ + e.updateProgressCounts(serverID, 1, 0) } - resultsMu.Unlock() }(gr) } wg.Wait() database.UpdateServerLastSync(serverID) - message := fmt.Sprintf("Sync completed: %d succeeded, %d failed", successCount, failedCount) + // Get final counts for message + p := e.GetProgress(serverID) + message := fmt.Sprintf("同步完成: %d 成功, %d 失败", p.SuccessCount, p.FailedCount) e.finishLog(syncLog, "success", message) + + // Mark as idle but keep final progress visible briefly + e.progressMu.Lock() + finalP := e.progress[serverID] + if finalP != nil { + finalP.Current = nil + finalP.Status = "completed" + } + e.progressMu.Unlock() + + // Clear after 30 seconds + go func() { + time.Sleep(30 * time.Second) + e.progressMu.Lock() + if p, ok := e.progress[serverID]; ok && p.Status == "completed" { + p.Status = "idle" + } + e.progressMu.Unlock() + }() + return nil } -func (e *Engine) cloneMirror(url, path string) error { +func (e *Engine) cloneMirror(cloneURL, token, path string) error { if _, err := os.Stat(path); err == nil { return nil } os.MkdirAll(filepath.Dir(path), 0755) - cmd := exec.Command("git", "clone", "--mirror", url, path) + authURL := injectToken(cloneURL, token) + cmd := exec.Command("git", "clone", "--mirror", authURL, path) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("clone failed: %w - %s", err, string(output)) @@ -157,6 +266,15 @@ func (e *Engine) cloneMirror(url, path string) error { return nil } +func injectToken(rawURL, token string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + u.User = url.UserPassword(token, "x-oauth-basic") + return u.String() +} + func (e *Engine) fetchMirror(path string) error { cmd := exec.Command("git", "--git-dir", path, "fetch", "--all", "--prune") output, err := cmd.CombinedOutput() diff --git a/main.go b/main.go index f276e41..0638d6a 100644 --- a/main.go +++ b/main.go @@ -101,16 +101,25 @@ func runServer(cfg *config.Config, engine *sync.Engine) error { // Serve embedded frontend distFS, err := fs.Sub(webFS, "web/dist") if err == nil { - // Serve assets - r.StaticFS("/assets", http.FS(distFS)) - // SPA fallback - serve index.html for non-API routes + fileServer := http.FileServer(http.FS(distFS)) r.NoRoute(func(c *gin.Context) { - if !strings.HasPrefix(c.Request.URL.Path, "/api") { - data, _ := webFS.ReadFile("web/dist/index.html") - c.Data(http.StatusOK, "text/html; charset=utf-8", data) - } else { + if strings.HasPrefix(c.Request.URL.Path, "/api") { c.JSON(http.StatusNotFound, gin.H{"error": "Not found"}) + return } + // Try to serve static file first + name := strings.TrimPrefix(c.Request.URL.Path, "/") + if name != "" { + f, err := distFS.Open(name) + if err == nil { + f.Close() + fileServer.ServeHTTP(c.Writer, c.Request) + return + } + } + // SPA fallback - serve index.html + data, _ := webFS.ReadFile("web/dist/index.html") + c.Data(http.StatusOK, "text/html; charset=utf-8", data) }) } @@ -124,10 +133,10 @@ func runServer(cfg *config.Config, engine *sync.Engine) error { protected.GET("/settings", handler.HandleGetSettings) protected.PUT("/settings", handler.HandleUpdateSettings) protected.GET("/servers", handler.HandleListServers) + protected.POST("/servers/test", handler.HandleTestConnection) protected.POST("/servers", handler.HandleCreateServer) protected.PUT("/servers/:id", handler.HandleUpdateServer) protected.DELETE("/servers/:id", handler.HandleDeleteServer) - protected.POST("/servers/:id/test", handler.HandleTestConnection) protected.GET("/servers/:id/repos", handler.HandleListRepos) protected.POST("/servers/:id/discover", handler.HandleDiscoverRepos) protected.POST("/servers/:id/sync", handler.HandleSyncServer(engine)) diff --git a/web/src/views/Dashboard.vue b/web/src/views/Dashboard.vue index ff1a936..f496d52 100644 --- a/web/src/views/Dashboard.vue +++ b/web/src/views/Dashboard.vue @@ -1,25 +1,25 @@