灾容datakeep项目推送
This commit is contained in:
5
.idea/.gitignore
generated
vendored
Normal file
5
.idea/.gitignore
generated
vendored
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
# 默认忽略的文件
|
||||||
|
/shelf/
|
||||||
|
/workspace.xml
|
||||||
|
# 基于编辑器的 HTTP 客户端请求
|
||||||
|
/httpRequests/
|
||||||
8
.idea/auto-save.iml
generated
Normal file
8
.idea/auto-save.iml
generated
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<module type="PYTHON_MODULE" version="4">
|
||||||
|
<component name="NewModuleRootManager">
|
||||||
|
<content url="file://$MODULE_DIR$" />
|
||||||
|
<orderEntry type="inheritedJdk" />
|
||||||
|
<orderEntry type="sourceFolder" forTests="false" />
|
||||||
|
</component>
|
||||||
|
</module>
|
||||||
6
.idea/inspectionProfiles/profiles_settings.xml
generated
Normal file
6
.idea/inspectionProfiles/profiles_settings.xml
generated
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
<component name="InspectionProjectProfileManager">
|
||||||
|
<settings>
|
||||||
|
<option name="USE_PROJECT_PROFILE" value="false" />
|
||||||
|
<version value="1.0" />
|
||||||
|
</settings>
|
||||||
|
</component>
|
||||||
8
.idea/modules.xml
generated
Normal file
8
.idea/modules.xml
generated
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="ProjectModuleManager">
|
||||||
|
<modules>
|
||||||
|
<module fileurl="file://$PROJECT_DIR$/.idea/auto-save.iml" filepath="$PROJECT_DIR$/.idea/auto-save.iml" />
|
||||||
|
</modules>
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
6
.idea/vcs.xml
generated
Normal file
6
.idea/vcs.xml
generated
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="VcsDirectoryMappings">
|
||||||
|
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
42
PROGRESS.md
Normal file
42
PROGRESS.md
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
# DataKeep 项目进度记录 (PROGRESS.md)
|
||||||
|
|
||||||
|
## 1. 项目愿景
|
||||||
|
实现一个轻量级、物理隔离的数据库容灾备份系统,运行在备份服务器上,主动从生产环境“拉取”数据并同步到本地的 MySQL、Redis 和 MinIO 副本中。
|
||||||
|
|
||||||
|
## 2. 核心决策 (2026-02-09)
|
||||||
|
- **同步方向**: Pull (从备份服务器主动连接生产服务器)。
|
||||||
|
- **同步频率**: 小时级 (通过前端 UI 配置 `sync_hour`)。
|
||||||
|
- **MySQL 同步**: `mysqldump` 全量逻辑导出 -> 本地导入。支持**自动建库**(`--databases`)与**结构变更自动覆盖**(`--add-drop-database`)。
|
||||||
|
- **Redis 同步**: `SCAN` + `DUMP` + `RESTORE` (键级同步),同步前 `FLUSHALL` 备份库。
|
||||||
|
- **MinIO 同步**: 增量镜像 (ETag + Size 对比),只增不删 (A方案)。
|
||||||
|
- **后端存储**: SQLite (SQLAlchemy) 存储实例配置与运行记录。
|
||||||
|
- **前端框架**: Vue 2 (使用 Vue-CLI 结构)。
|
||||||
|
- **部署方式**: Docker Compose 一键部署。
|
||||||
|
- **暴露端口**: 前端 Web 界面端口为 `12000`。
|
||||||
|
|
||||||
|
## 3. 任务进度清单
|
||||||
|
|
||||||
|
### 第一阶段:后端加固 [已完成]
|
||||||
|
- [x] 后端基础框架 (FastAPI + APScheduler)
|
||||||
|
- [x] MySQL 同步逻辑实现 (mysqldump)
|
||||||
|
- [x] Redis 键级同步逻辑 (SCAN/DUMP/RESTORE)
|
||||||
|
- [x] MinIO 增量同步逻辑 (SDK 模式,只增不删)
|
||||||
|
- [x] 集成 SQLite 存储 (SQLAlchemy) 替代 JSON
|
||||||
|
- [x] 完善实例级 RESTful CRUD API
|
||||||
|
|
||||||
|
### 第二阶段:前端工程化 [已完成]
|
||||||
|
- [x] 初始化 Vue-CLI 完整目录结构
|
||||||
|
- [x] 实现 Axios 封装与后端 API 对接
|
||||||
|
- [x] 实现实例列表、新增、编辑、删除功能
|
||||||
|
- [x] 实现小时级 (0-23) 调度选择 UI
|
||||||
|
- [x] 实现运行历史日志查看页面
|
||||||
|
|
||||||
|
### 第三阶段:容器化与交付 [已完成]
|
||||||
|
- [x] 编写后端 Dockerfile (含 mysql-client)
|
||||||
|
- [x] 编写前端 Dockerfile (多阶段构建 + Nginx)
|
||||||
|
- [x] 编写 docker-compose.yml (集成 MySQL/Redis/MinIO 备份实例)
|
||||||
|
- [x] 修改前端暴露端口为 12000
|
||||||
|
- [x] 编写 README.md 部署指南
|
||||||
|
|
||||||
|
---
|
||||||
|
*状态: 全部完成 | 更新时间: 2026-02-09*
|
||||||
65
README.md
Normal file
65
README.md
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
# DataKeep
|
||||||
|
|
||||||
|
一个运行在**备份服务器**上的容灾副本同步服务(Pull 模式):
|
||||||
|
|
||||||
|
- **备份服务器**上运行一套 MySQL/Redis/MinIO(作为容灾副本/隔离副本)。
|
||||||
|
- `datakeep-backend` 定时从**生产环境**直连拉取数据,并同步写入备份端的 MySQL/Redis/MinIO。
|
||||||
|
- 前端使用 **Vue2 + Vue CLI + ElementUI**,提供小时级(0-23)定时配置。
|
||||||
|
|
||||||
|
## 核心能力
|
||||||
|
|
||||||
|
- **MySQL 同步**:支持**自动建库**与**结构变更自动同步**(全量覆盖模式)。
|
||||||
|
- **Redis 同步**:键级全量同步 `SCAN + DUMP + RESTORE`,同步前自动 `FLUSHALL` 备份库。
|
||||||
|
- **MinIO 同步**:增量镜像,**只增不删**,确保生产端误删不传播。
|
||||||
|
- **配置存储**:使用 SQLite 保证数据一致性。
|
||||||
|
|
||||||
|
## 本地启动指南 (非 Docker)
|
||||||
|
|
||||||
|
如果你想在本地开发环境直接运行,请按以下步骤操作:
|
||||||
|
|
||||||
|
### 1. 后端 (FastAPI)
|
||||||
|
1. 进入 `backend` 目录。
|
||||||
|
2. 创建并激活虚拟环境:
|
||||||
|
```bash
|
||||||
|
python -m venv .venv
|
||||||
|
# Windows:
|
||||||
|
.\.venv\Scripts\activate
|
||||||
|
# Linux/macOS:
|
||||||
|
source .venv/bin/activate
|
||||||
|
```
|
||||||
|
3. 安装依赖:
|
||||||
|
```bash
|
||||||
|
pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
4. **前提条件**:确保本地已安装 `mysql-client`(支持 `mysqldump` 命令)。
|
||||||
|
5. 启动服务:
|
||||||
|
```bash
|
||||||
|
uvicorn app:app --host 0.0.0.0 --port 8000 --reload
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. 前端 (Vue CLI)
|
||||||
|
1. 进入 `frontend` 目录。
|
||||||
|
2. 安装依赖:
|
||||||
|
```bash
|
||||||
|
npm install
|
||||||
|
```
|
||||||
|
3. 启动开发服务器:
|
||||||
|
```bash
|
||||||
|
npm run serve
|
||||||
|
```
|
||||||
|
4. 访问地址:`http://localhost:8080`(默认会自动代理 API 请求到 8000 端口)。
|
||||||
|
|
||||||
|
## 生产部署 (Docker Compose)
|
||||||
|
|
||||||
|
在备份服务器上,确保已安装 Docker,然后执行:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d --build
|
||||||
|
```
|
||||||
|
访问地址:`http://服务器IP:12000`
|
||||||
|
|
||||||
|
## 配置说明
|
||||||
|
- **生产端 (Source)**:填写客户正在使用的生产环境数据库 IP 和账号。
|
||||||
|
- **备份端 (Target)**:
|
||||||
|
- 如果使用 Docker 部署,Host 建议填写容器名:`dr-mysql`, `dr-redis`, `dr-minio`。
|
||||||
|
- 如果本地部署,填写 `127.0.0.1` 及对应端口。
|
||||||
19
backend/Dockerfile
Normal file
19
backend/Dockerfile
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
FROM python:3.10-slim
|
||||||
|
|
||||||
|
# 安装同步所需的 MySQL 客户端工具
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
default-mysql-client \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
COPY app.py .
|
||||||
|
|
||||||
|
# 创建数据持久化目录
|
||||||
|
RUN mkdir -p /app/datakeep_data
|
||||||
|
|
||||||
|
EXPOSE 8000
|
||||||
|
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||||
292
backend/app.py
Normal file
292
backend/app.py
Normal file
@@ -0,0 +1,292 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
|
from apscheduler.triggers.cron import CronTrigger
|
||||||
|
from fastapi import FastAPI, HTTPException, BackgroundTasks, Depends
|
||||||
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
from minio import Minio
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from sqlalchemy import create_all_models, create_engine, Column, Integer, String, Boolean, DateTime, Text, ForeignKey
|
||||||
|
from sqlalchemy.ext.declarative import declarative_base
|
||||||
|
from sqlalchemy.orm import sessionmaker, Session
|
||||||
|
|
||||||
|
# 日志配置
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||||
|
logger = logging.getLogger("DataKeep")
|
||||||
|
|
||||||
|
DATA_DIR = Path(os.environ.get("DATAKEEP_DATA_DIR", "./datakeep_data")).resolve()
|
||||||
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
DB_PATH = DATA_DIR / "datakeep.db"
|
||||||
|
RUN_DIR = DATA_DIR / "runs"
|
||||||
|
RUN_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# SQLAlchemy 设置
|
||||||
|
engine = create_engine(f"sqlite:///{DB_PATH}", connect_args={"check_same_thread": False})
|
||||||
|
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||||
|
Base = declarative_base()
|
||||||
|
|
||||||
|
# --- 数据库模型 ---
|
||||||
|
|
||||||
|
class Instance(Base):
|
||||||
|
__tablename__ = "instances"
|
||||||
|
id = Column(String, primary_key=True, index=True)
|
||||||
|
name = Column(String)
|
||||||
|
enabled = Column(Boolean, default=True)
|
||||||
|
sync_hour = Column(Integer, default=2)
|
||||||
|
# 存储连接信息的 JSON 字符串
|
||||||
|
config_json = Column(Text)
|
||||||
|
|
||||||
|
class RunRecord(Base):
|
||||||
|
__tablename__ = "run_records"
|
||||||
|
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
||||||
|
instance_id = Column(String, index=True)
|
||||||
|
started_at = Column(DateTime, default=datetime.utcnow)
|
||||||
|
finished_at = Column(DateTime, nullable=True)
|
||||||
|
ok = Column(Boolean, default=False)
|
||||||
|
steps_json = Column(Text)
|
||||||
|
|
||||||
|
Base.metadata.create_all(bind=engine)
|
||||||
|
|
||||||
|
# --- Pydantic 模型 ---
|
||||||
|
|
||||||
|
class MySQLConn(BaseModel):
|
||||||
|
host: str
|
||||||
|
port: int = 3306
|
||||||
|
user: str
|
||||||
|
password: str
|
||||||
|
database: str
|
||||||
|
|
||||||
|
class RedisConn(BaseModel):
|
||||||
|
host: str
|
||||||
|
port: int = 6379
|
||||||
|
password: Optional[str] = None
|
||||||
|
|
||||||
|
class MinIOConn(BaseModel):
|
||||||
|
endpoint: str
|
||||||
|
access_key: str
|
||||||
|
secret_key: str
|
||||||
|
secure: bool = False
|
||||||
|
|
||||||
|
class InstanceSchema(BaseModel):
|
||||||
|
id: str
|
||||||
|
name: str
|
||||||
|
enabled: bool = True
|
||||||
|
sync_hour: int = Field(2, ge=0, le=23)
|
||||||
|
|
||||||
|
prod_mysql: Optional[MySQLConn] = None
|
||||||
|
dr_mysql: Optional[MySQLConn] = None
|
||||||
|
prod_redis: Optional[RedisConn] = None
|
||||||
|
dr_redis: Optional[RedisConn] = None
|
||||||
|
prod_minio: Optional[MinIOConn] = None
|
||||||
|
dr_minio: Optional[MinIOConn] = None
|
||||||
|
minio_buckets: List[str] = []
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
from_attributes = True
|
||||||
|
|
||||||
|
# --- 依赖项 ---
|
||||||
|
|
||||||
|
def get_db():
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
yield db
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
# --- 同步逻辑 (保持不变但适配新模型) ---
|
||||||
|
|
||||||
|
def mysql_sync(prod: MySQLConn, dr: MySQLConn):
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
dump_path = Path(td) / "dump.sql"
|
||||||
|
# 增加 --add-drop-database 确保目标端库级别也能同步更新/重建
|
||||||
|
dump_args = [
|
||||||
|
"mysqldump", f"-h{prod.host}", f"-P{prod.port}", f"-u{prod.user}",
|
||||||
|
"--databases", prod.database,
|
||||||
|
"--add-drop-database",
|
||||||
|
"--single-transaction",
|
||||||
|
"--set-gtid-purged=OFF",
|
||||||
|
"--column-statistics=0",
|
||||||
|
"--routines",
|
||||||
|
"--triggers"
|
||||||
|
]
|
||||||
|
env = {**os.environ, "MYSQL_PWD": prod.password}
|
||||||
|
with open(dump_path, "w", encoding="utf-8") as f:
|
||||||
|
p1 = subprocess.run(dump_args, stdout=f, stderr=subprocess.PIPE, text=True, env=env)
|
||||||
|
if p1.returncode != 0: raise RuntimeError(f"MySQL Dump Error: {p1.stderr}")
|
||||||
|
|
||||||
|
load_args = ["mysql", f"-h{dr.host}", f"-P{dr.port}", f"-u{dr.user}"]
|
||||||
|
env_dr = {**os.environ, "MYSQL_PWD": dr.password}
|
||||||
|
with open(dump_path, "r", encoding="utf-8") as f:
|
||||||
|
p2 = subprocess.run(load_args, stdin=f, stderr=subprocess.PIPE, text=True, env=env_dr)
|
||||||
|
if p2.returncode != 0: raise RuntimeError(f"MySQL Load Error: {p2.stderr}")
|
||||||
|
|
||||||
|
def redis_sync(prod: RedisConn, dr: RedisConn):
|
||||||
|
r_prod = redis.Redis(host=prod.host, port=prod.port, password=prod.password, decode_responses=False)
|
||||||
|
r_dr = redis.Redis(host=dr.host, port=dr.port, password=dr.password, decode_responses=False)
|
||||||
|
r_dr.flushall()
|
||||||
|
cursor = 0
|
||||||
|
count = 0
|
||||||
|
while True:
|
||||||
|
cursor, keys = r_prod.scan(cursor=cursor, count=1000)
|
||||||
|
for key in keys:
|
||||||
|
ttl = r_prod.ttl(key)
|
||||||
|
if ttl == -2: continue
|
||||||
|
px_ttl = ttl * 1000 if ttl > 0 else 0
|
||||||
|
val = r_prod.dump(key)
|
||||||
|
if val:
|
||||||
|
r_dr.restore(key, px_ttl, val, replace=True)
|
||||||
|
count += 1
|
||||||
|
if cursor == 0: break
|
||||||
|
return count
|
||||||
|
|
||||||
|
def minio_sync(prod: MinIOConn, dr: MinIOConn, buckets: List[str]):
|
||||||
|
client_p = Minio(prod.endpoint, access_key=prod.access_key, secret_key=prod.secret_key, secure=prod.secure)
|
||||||
|
client_d = Minio(dr.endpoint, access_key=dr.access_key, secret_key=dr.secret_key, secure=dr.secure)
|
||||||
|
synced = 0
|
||||||
|
for b in buckets:
|
||||||
|
if not client_d.bucket_exists(b): client_d.make_bucket(b)
|
||||||
|
for obj in client_p.list_objects(b, recursive=True):
|
||||||
|
try:
|
||||||
|
stat_d = client_d.stat_object(b, obj.object_name)
|
||||||
|
if stat_d.size == obj.size and stat_d.etag == obj.etag: continue
|
||||||
|
except: pass
|
||||||
|
res = client_p.get_object(b, obj.object_name)
|
||||||
|
try:
|
||||||
|
client_d.put_object(b, obj.object_name, res, obj.size, content_type=obj.content_type or "application/octet-stream")
|
||||||
|
synced += 1
|
||||||
|
finally:
|
||||||
|
res.close()
|
||||||
|
res.release_conn()
|
||||||
|
return synced
|
||||||
|
|
||||||
|
# --- 任务调度 ---
|
||||||
|
|
||||||
|
scheduler = BackgroundScheduler(timezone="Asia/Shanghai")
|
||||||
|
instance_locks: Dict[str, Lock] = {}
|
||||||
|
|
||||||
|
def run_instance_task(instance_id: str):
|
||||||
|
if instance_id not in instance_locks: instance_locks[instance_id] = Lock()
|
||||||
|
if not instance_locks[instance_id].acquire(blocking=False): return
|
||||||
|
|
||||||
|
db = SessionLocal()
|
||||||
|
record = RunRecord(instance_id=instance_id, started_at=datetime.utcnow(), steps_json="[]")
|
||||||
|
db.add(record)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
steps = []
|
||||||
|
ok = True
|
||||||
|
try:
|
||||||
|
inst_db = db.query(Instance).filter(Instance.id == instance_id).first()
|
||||||
|
if not inst_db: return
|
||||||
|
cfg = InstanceSchema.model_validate(json.loads(inst_db.config_json))
|
||||||
|
|
||||||
|
if cfg.prod_mysql and cfg.dr_mysql:
|
||||||
|
try:
|
||||||
|
mysql_sync(cfg.prod_mysql, cfg.dr_mysql)
|
||||||
|
steps.append({"name": "MySQL", "ok": True})
|
||||||
|
except Exception as e:
|
||||||
|
steps.append({"name": "MySQL", "ok": False, "error": str(e)})
|
||||||
|
ok = False
|
||||||
|
|
||||||
|
if ok and cfg.prod_redis and cfg.dr_redis:
|
||||||
|
try:
|
||||||
|
keys = redis_sync(cfg.prod_redis, cfg.dr_redis)
|
||||||
|
steps.append({"name": "Redis", "ok": True, "count": keys})
|
||||||
|
except Exception as e:
|
||||||
|
steps.append({"name": "Redis", "ok": False, "error": str(e)})
|
||||||
|
ok = False
|
||||||
|
|
||||||
|
if ok and cfg.prod_minio and cfg.dr_minio and cfg.minio_buckets:
|
||||||
|
try:
|
||||||
|
files = minio_sync(cfg.prod_minio, cfg.dr_minio, cfg.minio_buckets)
|
||||||
|
steps.append({"name": "MinIO", "ok": True, "count": files})
|
||||||
|
except Exception as e:
|
||||||
|
steps.append({"name": "MinIO", "ok": False, "error": str(e)})
|
||||||
|
ok = False
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
ok = False
|
||||||
|
steps.append({"name": "System", "ok": False, "error": str(e)})
|
||||||
|
finally:
|
||||||
|
record.finished_at = datetime.utcnow()
|
||||||
|
record.ok = ok
|
||||||
|
record.steps_json = json.dumps(steps)
|
||||||
|
db.commit()
|
||||||
|
db.close()
|
||||||
|
instance_locks[instance_id].release()
|
||||||
|
|
||||||
|
def reschedule_jobs():
|
||||||
|
scheduler.remove_all_jobs()
|
||||||
|
db = SessionLocal()
|
||||||
|
instances = db.query(Instance).filter(Instance.enabled == True).all()
|
||||||
|
for inst in instances:
|
||||||
|
scheduler.add_job(run_instance_task, CronTrigger(hour=inst.sync_hour, minute=0), args=[inst.id], id=inst.id)
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
# --- API ---
|
||||||
|
|
||||||
|
app = FastAPI(title="DataKeep")
|
||||||
|
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
def startup():
|
||||||
|
reschedule_jobs()
|
||||||
|
scheduler.start()
|
||||||
|
|
||||||
|
@app.get("/api/instances", response_model=List[InstanceSchema])
|
||||||
|
def list_instances(db: Session = Depends(get_db)):
|
||||||
|
insts = db.query(Instance).all()
|
||||||
|
return [InstanceSchema.model_validate(json.loads(i.config_json)) for i in insts]
|
||||||
|
|
||||||
|
@app.post("/api/instances")
|
||||||
|
def create_instance(inst: InstanceSchema, db: Session = Depends(get_db)):
|
||||||
|
db_inst = Instance(id=inst.id, name=inst.name, enabled=inst.enabled, sync_hour=inst.sync_hour, config_json=inst.model_dump_json())
|
||||||
|
db.add(db_inst)
|
||||||
|
db.commit()
|
||||||
|
reschedule_jobs()
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
@app.put("/api/instances/{id}")
|
||||||
|
def update_instance(id: str, inst: InstanceSchema, db: Session = Depends(get_db)):
|
||||||
|
db_inst = db.query(Instance).filter(Instance.id == id).first()
|
||||||
|
if not db_inst: raise HTTPException(404)
|
||||||
|
db_inst.name = inst.name
|
||||||
|
db_inst.enabled = inst.enabled
|
||||||
|
db_inst.sync_hour = inst.sync_hour
|
||||||
|
db_inst.config_json = inst.model_dump_json()
|
||||||
|
db.commit()
|
||||||
|
reschedule_jobs()
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
@app.delete("/api/instances/{id}")
|
||||||
|
def delete_instance(id: str, db: Session = Depends(get_db)):
|
||||||
|
db.query(Instance).filter(Instance.id == id).delete()
|
||||||
|
db.commit()
|
||||||
|
reschedule_jobs()
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
@app.get("/api/instances/{id}/runs")
|
||||||
|
def get_runs(id: str, db: Session = Depends(get_db)):
|
||||||
|
runs = db.query(RunRecord).filter(RunRecord.instance_id == id).order_by(RunRecord.started_at.desc()).limit(20).all()
|
||||||
|
return [{"id": r.id, "started_at": r.started_at, "finished_at": r.finished_at, "ok": r.ok, "steps": json.loads(r.steps_json)} for r in runs]
|
||||||
|
|
||||||
|
@app.post("/api/instances/{id}/run")
|
||||||
|
def trigger_run(id: str, bg: BackgroundTasks):
|
||||||
|
bg.add_task(run_instance_task, id)
|
||||||
|
return {"status": "triggered"}
|
||||||
|
|
||||||
|
@app.get("/api/health")
|
||||||
|
def health(): return {"status": "ok"}
|
||||||
10
backend/requirements.txt
Normal file
10
backend/requirements.txt
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
fastapi
|
||||||
|
uvicorn
|
||||||
|
pydantic
|
||||||
|
apscheduler
|
||||||
|
minio
|
||||||
|
python-multipart
|
||||||
|
redis
|
||||||
|
pymysql
|
||||||
|
sqlalchemy
|
||||||
|
|
||||||
76
docker-compose.yml
Normal file
76
docker-compose.yml
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
backend:
|
||||||
|
build: ./backend
|
||||||
|
container_name: datakeep-backend
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
- TZ=Asia/Shanghai
|
||||||
|
- DATAKEEP_DATA_DIR=/data
|
||||||
|
volumes:
|
||||||
|
- ./data/datakeep:/data
|
||||||
|
depends_on:
|
||||||
|
- dr-mysql
|
||||||
|
- dr-redis
|
||||||
|
- dr-minio
|
||||||
|
networks:
|
||||||
|
- datakeep-net
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
build: ./frontend
|
||||||
|
container_name: datakeep-frontend
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- "12000:80"
|
||||||
|
depends_on:
|
||||||
|
- backend
|
||||||
|
networks:
|
||||||
|
- datakeep-net
|
||||||
|
|
||||||
|
dr-mysql:
|
||||||
|
image: mysql:8.0
|
||||||
|
container_name: dr-mysql
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
MYSQL_ROOT_PASSWORD: rootpass
|
||||||
|
MYSQL_DATABASE: datakeep
|
||||||
|
volumes:
|
||||||
|
- ./data/mysql:/var/lib/mysql
|
||||||
|
ports:
|
||||||
|
- "3306:3306"
|
||||||
|
networks:
|
||||||
|
- datakeep-net
|
||||||
|
|
||||||
|
dr-redis:
|
||||||
|
image: redis:7-alpine
|
||||||
|
container_name: dr-redis
|
||||||
|
restart: unless-stopped
|
||||||
|
ports:
|
||||||
|
- "6379:6379"
|
||||||
|
volumes:
|
||||||
|
- ./data/redis:/data
|
||||||
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
|
networks:
|
||||||
|
- datakeep-net
|
||||||
|
|
||||||
|
dr-minio:
|
||||||
|
image: minio/minio:latest
|
||||||
|
container_name: dr-minio
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minioadmin
|
||||||
|
MINIO_ROOT_PASSWORD: minioadmin123
|
||||||
|
command: server /data --console-address ":9001"
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
|
- "9001:9001"
|
||||||
|
volumes:
|
||||||
|
- ./data/minio:/data
|
||||||
|
networks:
|
||||||
|
- datakeep-net
|
||||||
|
|
||||||
|
networks:
|
||||||
|
datakeep-net:
|
||||||
|
driver: bridge
|
||||||
|
|
||||||
14
frontend/Dockerfile
Normal file
14
frontend/Dockerfile
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
# build stage
|
||||||
|
FROM node:18-alpine AS build
|
||||||
|
WORKDIR /app
|
||||||
|
COPY package.json package-lock.json* ./
|
||||||
|
RUN npm install
|
||||||
|
COPY . .
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
# run stage
|
||||||
|
FROM nginx:1.25-alpine
|
||||||
|
COPY --from=build /app/dist /usr/share/nginx/html
|
||||||
|
COPY nginx.conf /etc/nginx/conf.d/default.conf
|
||||||
|
EXPOSE 80
|
||||||
|
|
||||||
16
frontend/nginx.conf
Normal file
16
frontend/nginx.conf
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
server {
|
||||||
|
listen 80;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
root /usr/share/nginx/html;
|
||||||
|
try_files $uri $uri/ /index.html;
|
||||||
|
}
|
||||||
|
|
||||||
|
location /api/ {
|
||||||
|
proxy_pass http://backend:8000/api/;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
20
frontend/package.json
Normal file
20
frontend/package.json
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"name": "datakeep-ui",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"serve": "vue-cli-service serve",
|
||||||
|
"build": "vue-cli-service build"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"axios": "^1.6.0",
|
||||||
|
"core-js": "^3.8.3",
|
||||||
|
"element-ui": "^2.15.14",
|
||||||
|
"vue": "^2.6.14"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@vue/cli-service": "~5.0.0",
|
||||||
|
"vue-template-compiler": "^2.6.14"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
216
frontend/src/App.vue
Normal file
216
frontend/src/App.vue
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
<template>
|
||||||
|
<div id="app" style="padding: 20px; background-color: #f0f2f5; min-height: 100vh;">
|
||||||
|
<el-card>
|
||||||
|
<div slot="header" style="display: flex; justify-content: space-between; align-items: center;">
|
||||||
|
<span style="font-size: 20px; font-weight: bold;">DataKeep 容灾备份系统</span>
|
||||||
|
<el-button type="primary" icon="el-icon-plus" @click="showCreate">新增备份实例</el-button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<el-table :data="instances" style="width: 100%">
|
||||||
|
<el-table-column prop="id" label="实例ID" width="120"></el-table-column>
|
||||||
|
<el-table-column prop="name" label="名称" width="180"></el-table-column>
|
||||||
|
<el-table-column label="同步时间" width="150">
|
||||||
|
<template slot-scope="scope">
|
||||||
|
每天 {{ scope.row.sync_hour }}:00
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="状态" width="100">
|
||||||
|
<template slot-scope="scope">
|
||||||
|
<el-tag :type="scope.row.enabled ? 'success' : 'info'">{{ scope.row.enabled ? '启用' : '禁用' }}</el-tag>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column label="操作">
|
||||||
|
<template slot-scope="scope">
|
||||||
|
<el-button size="mini" type="success" @click="runInstance(scope.row.id)">立即同步</el-button>
|
||||||
|
<el-button size="mini" @click="showEdit(scope.row)">编辑</el-button>
|
||||||
|
<el-button size="mini" type="info" @click="viewHistory(scope.row.id)">历史</el-button>
|
||||||
|
<el-button size="mini" type="danger" @click="deleteInstance(scope.row.id)">删除</el-button>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
</el-table>
|
||||||
|
</el-card>
|
||||||
|
|
||||||
|
<!-- 编辑/新增对话框 -->
|
||||||
|
<el-dialog :title="isEdit ? '编辑实例' : '新增实例'" :visible.sync="dialogVisible" width="70%">
|
||||||
|
<el-form :model="form" label-width="120px" size="small">
|
||||||
|
<el-row :gutter="20">
|
||||||
|
<el-col :span="12">
|
||||||
|
<el-form-item label="实例ID"><el-input v-model="form.id" :disabled="isEdit"></el-input></el-form-item>
|
||||||
|
</el-col>
|
||||||
|
<el-col :span="12">
|
||||||
|
<el-form-item label="名称"><el-input v-model="form.name"></el-input></el-form-item>
|
||||||
|
</el-col>
|
||||||
|
</el-row>
|
||||||
|
<el-row :gutter="20">
|
||||||
|
<el-col :span="12">
|
||||||
|
<el-form-item label="每天同步时间">
|
||||||
|
<el-select v-model="form.sync_hour" placeholder="选择小时" style="width: 100%">
|
||||||
|
<el-option v-for="h in 24" :key="h-1" :label="`${h-1}:00`" :value="h-1"></el-option>
|
||||||
|
</el-select>
|
||||||
|
</el-form-item>
|
||||||
|
</el-col>
|
||||||
|
<el-col :span="12">
|
||||||
|
<el-form-item label="是否启用">
|
||||||
|
<el-switch v-model="form.enabled"></el-switch>
|
||||||
|
</el-form-item>
|
||||||
|
</el-col>
|
||||||
|
</el-row>
|
||||||
|
|
||||||
|
<el-tabs type="border-card" style="margin-top: 20px;">
|
||||||
|
<el-tab-pane label="MySQL (必选)">
|
||||||
|
<el-divider content-position="left">生产端 (源)</el-divider>
|
||||||
|
<el-row :gutter="10">
|
||||||
|
<el-col :span="16"><el-form-item label="Host"><el-input v-model="form.prod_mysql.host"></el-input></el-form-item></el-col>
|
||||||
|
<el-col :span="8"><el-form-item label="Port"><el-input-number v-model="form.prod_mysql.port" :controls="false" style="width: 100%"></el-input-number></el-form-item></el-col>
|
||||||
|
</el-row>
|
||||||
|
<el-row :gutter="10">
|
||||||
|
<el-col :span="8"><el-form-item label="User"><el-input v-model="form.prod_mysql.user"></el-input></el-form-item></el-col>
|
||||||
|
<el-col :span="8"><el-form-item label="Pass"><el-input v-model="form.prod_mysql.password" show-password></el-input></el-form-item></el-col>
|
||||||
|
<el-col :span="8"><el-form-item label="DB"><el-input v-model="form.prod_mysql.database"></el-input></el-form-item></el-col>
|
||||||
|
</el-row>
|
||||||
|
<el-divider content-position="left">备份端 (目标)</el-divider>
|
||||||
|
<el-row :gutter="10">
|
||||||
|
<el-col :span="16"><el-form-item label="Host"><el-input v-model="form.dr_mysql.host" placeholder="容器名: dr-mysql"></el-input></el-form-item></el-col>
|
||||||
|
<el-col :span="8"><el-form-item label="Port"><el-input-number v-model="form.dr_mysql.port" :controls="false" style="width: 100%"></el-input-number></el-form-item></el-col>
|
||||||
|
</el-row>
|
||||||
|
</el-tab-pane>
|
||||||
|
|
||||||
|
<el-tab-pane label="Redis (可选)">
|
||||||
|
<el-row :gutter="20">
|
||||||
|
<el-col :span="12">
|
||||||
|
<el-divider content-position="left">生产端 (源)</el-divider>
|
||||||
|
<el-form-item label="Host"><el-input v-model="form.prod_redis.host"></el-input></el-form-item>
|
||||||
|
<el-form-item label="Port"><el-input-number v-model="form.prod_redis.port" :controls="false" style="width: 100%"></el-input-number></el-form-item>
|
||||||
|
<el-form-item label="Pass"><el-input v-model="form.prod_redis.password" show-password></el-input></el-form-item>
|
||||||
|
</el-col>
|
||||||
|
<el-col :span="12">
|
||||||
|
<el-divider content-position="left">备份端 (目标)</el-divider>
|
||||||
|
<el-form-item label="Host"><el-input v-model="form.dr_redis.host" placeholder="容器名: dr-redis"></el-input></el-form-item>
|
||||||
|
<el-form-item label="Port"><el-input-number v-model="form.dr_redis.port" :controls="false" style="width: 100%"></el-input-number></el-form-item>
|
||||||
|
<el-form-item label="Pass"><el-input v-model="form.dr_redis.password" show-password></el-input></el-form-item>
|
||||||
|
</el-col>
|
||||||
|
</el-row>
|
||||||
|
</el-tab-pane>
|
||||||
|
|
||||||
|
<el-tab-pane label="MinIO (可选)">
|
||||||
|
<el-divider content-position="left">连接配置</el-divider>
|
||||||
|
<el-row :gutter="10">
|
||||||
|
<el-col :span="12"><el-form-item label="生产端 Endpoint"><el-input v-model="form.prod_minio.endpoint"></el-input></el-form-item></el-col>
|
||||||
|
<el-col :span="12"><el-form-item label="备份端 Endpoint"><el-input v-model="form.dr_minio.endpoint" placeholder="容器名: dr-minio:9000"></el-input></el-form-item></el-col>
|
||||||
|
</el-row>
|
||||||
|
<el-row :gutter="10">
|
||||||
|
<el-col :span="12"><el-form-item label="Buckets (逗号分隔)"><el-input v-model="bucketsStr" placeholder="bucket1,bucket2"></el-input></el-form-item></el-col>
|
||||||
|
</el-row>
|
||||||
|
</el-tab-pane>
|
||||||
|
</el-tabs>
|
||||||
|
</el-form>
|
||||||
|
<div slot="footer">
|
||||||
|
<el-button @click="dialogVisible = false">取消</el-button>
|
||||||
|
<el-button type="primary" @click="submitForm">确定</el-button>
|
||||||
|
</div>
|
||||||
|
</el-dialog>
|
||||||
|
|
||||||
|
<!-- 历史记录 -->
|
||||||
|
<el-drawer title="同步历史记录" :visible.sync="historyVisible" size="50%">
|
||||||
|
<div style="padding: 20px;">
|
||||||
|
<el-timeline>
|
||||||
|
<el-timeline-item v-for="run in runHistory" :key="run.id" :timestamp="run.started_at" :type="run.ok ? 'success' : 'danger'">
|
||||||
|
<el-card>
|
||||||
|
<h4>结果: {{ run.ok ? '成功' : '失败' }} (耗时: {{ calcDuration(run) }})</h4>
|
||||||
|
<div v-for="step in run.steps" :key="step.name">
|
||||||
|
<p><b>{{ step.name }}:</b> {{ step.ok ? 'OK' : 'ERROR: ' + step.error }} {{ step.count ? '(数量: ' + step.count + ')' : '' }}</p>
|
||||||
|
</div>
|
||||||
|
</el-card>
|
||||||
|
</el-timeline-item>
|
||||||
|
</el-timeline>
|
||||||
|
</div>
|
||||||
|
</el-drawer>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
export default {
|
||||||
|
data() {
|
||||||
|
return {
|
||||||
|
instances: [],
|
||||||
|
dialogVisible: false,
|
||||||
|
historyVisible: false,
|
||||||
|
isEdit: false,
|
||||||
|
runHistory: [],
|
||||||
|
bucketsStr: '',
|
||||||
|
form: {
|
||||||
|
id: '', name: '', enabled: true, sync_hour: 2,
|
||||||
|
prod_mysql: { host: '', port: 3306, user: '', password: '', database: '' },
|
||||||
|
dr_mysql: { host: '', port: 3306, user: '', password: '', database: '' },
|
||||||
|
prod_redis: { host: '', port: 6379, password: '' },
|
||||||
|
dr_redis: { host: '', port: 6379, password: '' },
|
||||||
|
prod_minio: { endpoint: '', access_key: '', secret_key: '', secure: false },
|
||||||
|
dr_minio: { endpoint: '', access_key: '', secret_key: '', secure: false },
|
||||||
|
minio_buckets: []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
mounted() { this.refresh(); },
|
||||||
|
methods: {
|
||||||
|
async refresh() {
|
||||||
|
const res = await this.$http.get('/instances');
|
||||||
|
this.instances = res.data;
|
||||||
|
},
|
||||||
|
showCreate() {
|
||||||
|
this.isEdit = false;
|
||||||
|
this.resetForm();
|
||||||
|
this.dialogVisible = true;
|
||||||
|
},
|
||||||
|
showEdit(row) {
|
||||||
|
this.isEdit = true;
|
||||||
|
this.form = JSON.parse(JSON.stringify(row));
|
||||||
|
this.bucketsStr = (this.form.minio_buckets || []).join(',');
|
||||||
|
this.dialogVisible = true;
|
||||||
|
},
|
||||||
|
resetForm() {
|
||||||
|
this.form = {
|
||||||
|
id: '', name: '', enabled: true, sync_hour: 2,
|
||||||
|
prod_mysql: { host: '', port: 3306, user: '', password: '', database: '' },
|
||||||
|
dr_mysql: { host: '', port: 3306, user: '', password: '', database: '' },
|
||||||
|
prod_redis: { host: '', port: 6379, password: '' },
|
||||||
|
dr_redis: { host: '', port: 6379, password: '' },
|
||||||
|
prod_minio: { endpoint: '', access_key: '', secret_key: '', secure: false },
|
||||||
|
dr_minio: { endpoint: '', access_key: '', secret_key: '', secure: false },
|
||||||
|
minio_buckets: []
|
||||||
|
};
|
||||||
|
this.bucketsStr = '';
|
||||||
|
},
|
||||||
|
async submitForm() {
|
||||||
|
this.form.minio_buckets = this.bucketsStr ? this.bucketsStr.split(',').map(s => s.trim()) : [];
|
||||||
|
if (this.isEdit) {
|
||||||
|
await this.$http.put(`/instances/${this.form.id}`, this.form);
|
||||||
|
} else {
|
||||||
|
await this.$http.post('/instances', this.form);
|
||||||
|
}
|
||||||
|
this.dialogVisible = false;
|
||||||
|
this.refresh();
|
||||||
|
this.$message.success('保存成功');
|
||||||
|
},
|
||||||
|
async runInstance(id) {
|
||||||
|
await this.$http.post(`/instances/${id}/run`);
|
||||||
|
this.$message.success('已触发同步任务');
|
||||||
|
},
|
||||||
|
async deleteInstance(id) {
|
||||||
|
await this.$confirm('确定删除此实例吗?');
|
||||||
|
await this.$http.delete(`/instances/${id}`);
|
||||||
|
this.refresh();
|
||||||
|
},
|
||||||
|
async viewHistory(id) {
|
||||||
|
const res = await this.$http.get(`/instances/${id}/runs`);
|
||||||
|
this.runHistory = res.data;
|
||||||
|
this.historyVisible = true;
|
||||||
|
},
|
||||||
|
calcDuration(run) {
|
||||||
|
if (!run.finished_at) return '进行中';
|
||||||
|
const d = new Date(run.finished_at) - new Date(run.started_at);
|
||||||
|
return Math.round(d / 1000) + 's';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
21
frontend/src/main.js
Normal file
21
frontend/src/main.js
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
import Vue from 'vue'
|
||||||
|
import App from './App.vue'
|
||||||
|
import ElementUI from 'element-ui'
|
||||||
|
import 'element-ui/lib/theme-chalk/index.css'
|
||||||
|
import axios from 'axios'
|
||||||
|
|
||||||
|
Vue.use(ElementUI)
|
||||||
|
Vue.config.productionTip = false
|
||||||
|
|
||||||
|
// 创建 axios 实例
|
||||||
|
const http = axios.create({
|
||||||
|
baseURL: process.env.VUE_APP_API_BASE || '/api',
|
||||||
|
timeout: 10000
|
||||||
|
})
|
||||||
|
|
||||||
|
Vue.prototype.$http = http
|
||||||
|
|
||||||
|
new Vue({
|
||||||
|
render: h => h(App),
|
||||||
|
}).$mount('#app')
|
||||||
|
|
||||||
12
frontend/vue.config.js
Normal file
12
frontend/vue.config.js
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
module.exports = {
|
||||||
|
devServer: {
|
||||||
|
proxy: {
|
||||||
|
'/api': {
|
||||||
|
target: 'http://localhost:8000',
|
||||||
|
changeOrigin: true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
transpileDependencies: true
|
||||||
|
}
|
||||||
|
|
||||||
Reference in New Issue
Block a user