From 751e4d780feeea622177d3fd6426a2beb6a06c48 Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Wed, 21 Oct 2026 16:51:08 +0800 Subject: [PATCH 1/2] fix: preserve persistent data in backups --- astrbot/core/backup/constants.py | 14 ++ astrbot/core/backup/exporter.py | 5 + astrbot/core/backup/importer.py | 22 ++- docs/en/use/webui.md | 2 + docs/zh/use/webui.md | 2 + tests/test_backup.py | 262 ++++++++++++++++++++++++++++++- 6 files changed, 305 insertions(+), 2 deletions(-) diff --git a/astrbot/core/backup/constants.py b/astrbot/core/backup/constants.py index d7a190c91a2..f38b67c915f 100644 --- a/astrbot/core/backup/constants.py +++ b/astrbot/core/backup/constants.py @@ -6,19 +6,23 @@ from sqlmodel import SQLModel from astrbot.core.db.po import ( + ApiKey, Attachment, ChatUIProject, CommandConfig, CommandConflict, ConversationV2, CronJob, + DashboardTrustedDevice, Persona, PersonaFolder, PlatformMessageHistory, PlatformSession, PlatformStat, Preference, + ProviderStat, SessionProjectRelation, + UmoAlias, WebChatThread, ) from astrbot.core.knowledge_base.models import ( @@ -34,6 +38,7 @@ get_astrbot_t2i_templates_path, get_astrbot_temp_path, get_astrbot_webchat_path, + get_astrbot_workspaces_path, ) # ============================================================ @@ -43,6 +48,7 @@ # 主数据库模型类映射 MAIN_DB_MODELS: dict[str, type[SQLModel]] = { "platform_stats": PlatformStat, + "provider_stats": ProviderStat, "conversations": ConversationV2, "cron_jobs": CronJob, "personas": Persona, @@ -56,6 +62,13 @@ "attachments": Attachment, "command_configs": CommandConfig, "command_conflicts": CommandConflict, + "umo_aliases": UmoAlias, + "api_keys": ApiKey, +} + +# Device trust must be re-established after restore, never copied from a backup. +EXCLUDED_MAIN_DB_MODELS: dict[str, type[SQLModel]] = { + "dashboard_trusted_devices": DashboardTrustedDevice, } # 知识库元数据模型类映射 @@ -82,6 +95,7 @@ def get_backup_directories() -> dict[str, str]: "webchat": get_astrbot_webchat_path(), # WebChat 数据 "temp": get_astrbot_temp_path(), # 临时文件 "skills": get_astrbot_skills_path(), # Skills + "workspaces": get_astrbot_workspaces_path(), # Session and project files } diff --git a/astrbot/core/backup/exporter.py b/astrbot/core/backup/exporter.py index a9223759989..2bd0c30178b 100644 --- a/astrbot/core/backup/exporter.py +++ b/astrbot/core/backup/exporter.py @@ -165,6 +165,11 @@ async def export_all( config_content = f.read() zf.writestr("config/cmd_config.json", config_content) self._add_checksum("config/cmd_config.json", config_content) + mcp_config_path = Path(get_astrbot_data_path()) / "mcp_server.json" + if mcp_config_path.is_file(): + mcp_content = mcp_config_path.read_bytes() + zf.writestr("config/mcp_server.json", mcp_content) + self._add_checksum("config/mcp_server.json", mcp_content) if progress_callback: await progress_callback("config", 100, 100, "配置文件导出完成") diff --git a/astrbot/core/backup/importer.py b/astrbot/core/backup/importer.py index 7d30d27c397..70de7b46d22 100644 --- a/astrbot/core/backup/importer.py +++ b/astrbot/core/backup/importer.py @@ -30,6 +30,7 @@ # 从共享常量模块导入 from .constants import ( + EXCLUDED_MAIN_DB_MODELS, KB_METADATA_MODELS, MAIN_DB_MODELS, get_backup_directories, @@ -471,6 +472,22 @@ async def import_all( except Exception as e: result.add_warning(f"导入配置文件失败: {e}") + if "config/mcp_server.json" in zf.namelist(): + try: + mcp_config_path = ( + Path(get_astrbot_data_path()) / "mcp_server.json" + ) + mcp_config_path.parent.mkdir(parents=True, exist_ok=True) + if mcp_config_path.exists(): + shutil.copy2( + mcp_config_path, + mcp_config_path.with_suffix(".json.bak"), + ) + mcp_config_path.write_bytes(zf.read("config/mcp_server.json")) + result.imported_files["mcp_config"] = 1 + except Exception as e: + result.add_warning(f"Failed to restore MCP configuration: {e}") + if progress_callback: await progress_callback("config", 100, 100, "配置文件导入完成") @@ -532,7 +549,10 @@ async def _clear_main_db(self) -> None: """清空主数据库所有表""" async with self.main_db.get_db() as session: async with session.begin(): - for table_name, model_class in MAIN_DB_MODELS.items(): + # Replace restores also invalidate device trust on the target instance. + for table_name, model_class in ( + MAIN_DB_MODELS | EXCLUDED_MAIN_DB_MODELS + ).items(): try: await session.execute(delete(model_class)) logger.debug(f"已清空表 {table_name}") diff --git a/docs/en/use/webui.md b/docs/en/use/webui.md index 3cb1fecf036..e65e4664f4f 100644 --- a/docs/en/use/webui.md +++ b/docs/en/use/webui.md @@ -135,6 +135,8 @@ The page title and section navigation stay fixed while only the settings pane on System configuration changes save automatically. Check for a successful save message and restart AstrBot if the page indicates that a restart is required. +Backups under `Maintenance` include session aliases, model usage statistics, OpenAPI access keys, MCP server configuration, and session and project files under `data/workspaces/`. Custom workspace directories outside `data/workspaces/` require a separate backup. TOTP trusted devices are excluded; replacing data from a backup clears existing device trust and requires device verification again. Older backups cannot recover data they did not include: export a new backup from the original instance before migrating. If a backup has no MCP configuration, the target instance's MCP configuration is kept. + ## Plugins Select `Extensions` in the sidebar. The top tabs are `Plugins`, `Skills`, `MCP Servers`, and `Handlers`. Within `Plugins`, switch between `Installed` and `AstrBot Plugin Market` to view local and market plugins. diff --git a/docs/zh/use/webui.md b/docs/zh/use/webui.md index 7420d67ab2e..b84b448dd12 100644 --- a/docs/zh/use/webui.md +++ b/docs/zh/use/webui.md @@ -135,6 +135,8 @@ ChatUI 支持以下常用能力： 系统配置修改后自动保存，请确认保存成功提示；如果页面提示需要重启，再按提示重启 AstrBot。 +`维护` 中的备份包含会话别名、模型用量统计、OpenAPI 访问密钥、MCP 服务配置，以及 `data/workspaces/` 下的会话和项目文件。位于 `data/workspaces/` 之外的自定义工作区需要单独备份。TOTP 受信任设备记录不纳入备份；替换恢复时会清除现有设备信任，需要重新验证设备。旧备份无法恢复未包含的数据，迁移前请从原实例重新导出备份。如果备份中没有 MCP 配置，则保留目标实例现有的 MCP 配置。 + ## 插件 点击左栏 `插件`，顶部可切换 `插件`、`技能`、`MCP` 和 `管理行为`。在 `插件` 标签内，通过 `已安装` 与 `插件市场` 切换查看本地插件和市场插件。 diff --git a/tests/test_backup.py b/tests/test_backup.py index 0671aba7269..31d0ecf8ec9 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -1,5 +1,6 @@ """备份功能单元测试""" +import hashlib import json import os import re @@ -9,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from sqlmodel import SQLModel, select from astrbot.core.backup import ( BACKUP_MANIFEST_VERSION, @@ -16,6 +18,7 @@ MAIN_DB_MODELS, ImportPreCheckResult, ) +from astrbot.core.backup.constants import EXCLUDED_MAIN_DB_MODELS from astrbot.core.backup.exporter import AstrBotExporter from astrbot.core.backup.importer import ( PLATFORM_STATS_INVALID_COUNT_WARN_LIMIT, @@ -25,8 +28,13 @@ _get_major_version, ) from astrbot.core.config.default import VERSION +from astrbot.core.db import po from astrbot.core.db.po import ( + ApiKey, ConversationV2, + DashboardTrustedDevice, + ProviderStat, + UmoAlias, ) from astrbot.core.db.sqlite import SQLiteDatabase from astrbot.core.utils.upload import UploadTooLargeError @@ -1017,6 +1025,21 @@ def test_main_db_models_not_empty(self): """测试主数据库模型映射非空""" assert len(MAIN_DB_MODELS) > 0 + def test_all_main_db_models_have_a_backup_policy(self): + """Require each main database table to be backed up or explicitly excluded.""" + models = { + model.__tablename__: model + for model in vars(po).values() + if isinstance(model, type) + and issubclass(model, SQLModel) + and "__table__" in vars(model) + } + assert MAIN_DB_MODELS.keys().isdisjoint(EXCLUDED_MAIN_DB_MODELS) + assert models == MAIN_DB_MODELS | EXCLUDED_MAIN_DB_MODELS + assert EXCLUDED_MAIN_DB_MODELS == { + "dashboard_trusted_devices": DashboardTrustedDevice, + } + def test_main_db_models_contain_expected_tables(self): """测试主数据库模型映射包含预期的表""" expected_tables = [ @@ -1049,6 +1072,241 @@ def test_kb_metadata_models_contain_expected_tables(self): class TestBackupIntegration: """备份集成测试""" + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("model", "data"), + [ + ( + UmoAlias, + { + "umo": "qq:GroupMessage:123", + "creator_sender_id": "admin", + "auto_name": "Original group name", + "user_alias": "Team chat", + }, + ), + ( + ProviderStat, + { + "umo": "qq:GroupMessage:123", + "conversation_id": "conversation-1", + "provider_id": "provider-1", + "provider_model": "model-1", + "status": "completed", + "token_input_other": 120, + "token_input_cached": 80, + "token_output": 40, + "start_time": 10.0, + "end_time": 12.5, + "time_to_first_token": 0.5, + }, + ), + ( + ApiKey, + { + "name": "External client", + "key_id": "key-1", + "key_hash": "a" * 64, + "key_prefix": "ab_test", + "scopes": ["chat", "sessions"], + "created_by": "admin", + "last_used_at": datetime(2026, 1, 1), + "expires_at": datetime(2099, 1, 1), + }, + ), + ( + ApiKey, + { + "name": "Revoked client", + "key_id": "key-2", + "key_hash": "b" * 64, + "key_prefix": "ab_revoked", + "scopes": [], + "created_by": "admin", + "revoked_at": datetime(2026, 1, 1), + }, + ), + ], + ids=["umo-alias", "provider-stats", "api-key", "revoked-api-key"], + ) + async def test_persistent_models_roundtrip( + self, tmp_path, monkeypatch, model, data + ): + """Preserve persistent records and replace existing target rows. + + Args: + tmp_path: Temporary directory for SQLite databases and the archive. + monkeypatch: Fixture used to isolate the AstrBot data directory. + model: Model whose backup behavior is under test. + data: Persisted field values to preserve through the archive. + """ + monkeypatch.setenv("ASTRBOT_ROOT", str(tmp_path)) + source = SQLiteDatabase(str(tmp_path / "source.db")) + target = SQLiteDatabase(str(tmp_path / "target.db")) + try: + async with source.get_db() as session: + async with session.begin(): + session.add(model(**data)) + async with source.get_db() as session: + expected = ( + (await session.execute(select(model))).scalar_one().model_dump() + ) + async with target.get_db() as session: + async with session.begin(): + primary_key = next(iter(model.__table__.primary_key)).name + session.add(model(**(data | {primary_key: 999}))) + exporter = AstrBotExporter( + main_db=source, config_path=str(tmp_path / "missing-config.json") + ) + zip_path = await exporter.export_all(output_dir=str(tmp_path / "backups")) + + result = await AstrBotImporter(main_db=target).import_all(zip_path) + assert result.success, result.errors + assert result.imported_tables[model.__tablename__] == 1 + async with target.get_db() as session: + restored = (await session.execute(select(model))).scalars().all() + assert [row.model_dump() for row in restored] == [expected] + if model is ApiKey: + active = await target.get_active_api_key_by_hash(data["key_hash"]) + assert (active is None) == (data.get("revoked_at") is not None) + finally: + await source.engine.dispose() + await target.engine.dispose() + + @pytest.mark.asyncio + async def test_device_trust_is_excluded_and_cleared(self, tmp_path, monkeypatch): + """Exclude source device trust and invalidate target trust on replace restore. + + Args: + tmp_path: Temporary directory for databases and the archive. + monkeypatch: Fixture used to isolate the AstrBot data directory. + """ + monkeypatch.setenv("ASTRBOT_ROOT", str(tmp_path)) + source = SQLiteDatabase(str(tmp_path / "source.db")) + target = SQLiteDatabase(str(tmp_path / "target.db")) + try: + for database in (source, target): + async with database.get_db() as session: + async with session.begin(): + session.add( + DashboardTrustedDevice( + token_hash="device-token", + totp_secret_hash="totp-secret", + expires_at=datetime(2099, 1, 1), + ) + ) + exporter = AstrBotExporter( + main_db=source, config_path=str(tmp_path / "missing-config.json") + ) + zip_path = await exporter.export_all(output_dir=str(tmp_path / "backups")) + with zipfile.ZipFile(zip_path) as archive: + main_data = json.loads(archive.read("databases/main_db.json")) + assert "dashboard_trusted_devices" not in main_data + + result = await AstrBotImporter(main_db=target).import_all(zip_path) + assert result.success, result.errors + async with target.get_db() as session: + assert ( + not (await session.execute(select(DashboardTrustedDevice))) + .scalars() + .all() + ) + finally: + await source.engine.dispose() + await target.engine.dispose() + + @pytest.mark.asyncio + @pytest.mark.parametrize("include_mcp", [True, False], ids=["mcp", "legacy"]) + @pytest.mark.parametrize( + "existing_target", [True, False], ids=["existing", "fresh"] + ) + async def test_mcp_and_workspaces_roundtrip( + self, tmp_path, monkeypatch, include_mcp, existing_target + ): + """Restore configuration and workspace files across distinct AstrBot roots. + + Args: + tmp_path: Temporary directory for both instances and their databases. + monkeypatch: Fixture used to switch between instance roots. + include_mcp: Whether the backup contains an MCP configuration. + existing_target: Whether the target already has configuration and files. + """ + source_root = tmp_path / "source" + target_root = tmp_path / "target" + source_data = source_root / "data" + source_data.mkdir(parents=True) + mcp_content = json.dumps( + { + "mcpServers": { + "search": {"url": "https://example.com/mcp", "enabled": False} + } + }, + indent=2, + ).encode("utf-8") + if include_mcp: + (source_data / "mcp_server.json").write_bytes(mcp_content) + workspace_files = { + "project_demo/report.txt": b"Project report", + "session/skills/example/SKILL.md": b"# Workspace skill", + "session/output.bin": bytes(range(256)), + } + for relative_path, content in workspace_files.items(): + file_path = source_data / "workspaces" / relative_path + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_bytes(content) + target_data = target_root / "data" + target_mcp = target_data / "mcp_server.json" + if existing_target: + (target_data / "workspaces").mkdir(parents=True) + (target_data / "workspaces" / "stale.txt").write_text("Old workspace") + target_mcp.write_bytes(b'{"mcpServers": {}}') + + source = SQLiteDatabase(str(tmp_path / "source.db")) + target = SQLiteDatabase(str(tmp_path / "target.db")) + try: + monkeypatch.setenv("ASTRBOT_ROOT", str(source_root)) + exporter = AstrBotExporter( + main_db=source, config_path=str(source_data / "cmd_config.json") + ) + zip_path = await exporter.export_all(output_dir=str(tmp_path / "backups")) + with zipfile.ZipFile(zip_path) as archive: + manifest = json.loads(archive.read("manifest.json")) + assert "workspaces" in manifest["directories"] + assert ("config/mcp_server.json" in archive.namelist()) == include_mcp + if include_mcp: + assert manifest["checksums"]["config/mcp_server.json"] == ( + "sha256:" + hashlib.sha256(mcp_content).hexdigest() + ) + + monkeypatch.setenv("ASTRBOT_ROOT", str(target_root)) + result = await AstrBotImporter( + main_db=target, config_path=str(target_data / "cmd_config.json") + ).import_all(zip_path) + assert result.success, result.errors + assert not result.warnings + assert result.imported_directories["workspaces"] == len(workspace_files) + for relative_path, content in workspace_files.items(): + assert ( + target_data / "workspaces" / relative_path + ).read_bytes() == content + assert not (target_data / "workspaces" / "stale.txt").exists() + if include_mcp: + assert target_mcp.read_bytes() == mcp_content + assert result.imported_files["mcp_config"] == 1 + if existing_target: + assert ( + target_mcp.with_suffix(".json.bak").read_bytes() + == b'{"mcpServers": {}}' + ) + elif existing_target: + assert target_mcp.read_bytes() == b'{"mcpServers": {}}' + assert not target_mcp.with_suffix(".json.bak").exists() + else: + assert not target_mcp.exists() + finally: + await source.engine.dispose() + await target.engine.dispose() + @pytest.mark.asyncio async def test_cron_jobs_roundtrip(self, tmp_path): """Preserve a scheduled task through backup export and import.""" @@ -1068,7 +1326,9 @@ async def test_cron_jobs_roundtrip(self, tmp_path): with patch( "astrbot.core.backup.exporter.get_backup_directories", return_value={} ): - zip_path = await exporter.export_all(output_dir=str(tmp_path / "backups")) + zip_path = await exporter.export_all( + output_dir=str(tmp_path / "backups") + ) result = await AstrBotImporter(main_db=target).import_all(zip_path) assert result.success, result.errors From 0c27191347b290c3ffab09e21c7b483eecaa0f9e Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Wed, 21 Oct 2026 18:46:31 +0800 Subject: [PATCH 2/2] fix: exclude workspaces from backup changes --- astrbot/core/backup/constants.py | 2 -- docs/en/use/webui.md | 2 +- docs/zh/use/webui.md | 2 +- tests/test_backup.py | 25 ++++--------------------- 4 files changed, 6 insertions(+), 25 deletions(-) diff --git a/astrbot/core/backup/constants.py b/astrbot/core/backup/constants.py index f38b67c915f..b3863dde93b 100644 --- a/astrbot/core/backup/constants.py +++ b/astrbot/core/backup/constants.py @@ -38,7 +38,6 @@ get_astrbot_t2i_templates_path, get_astrbot_temp_path, get_astrbot_webchat_path, - get_astrbot_workspaces_path, ) # ============================================================ @@ -95,7 +94,6 @@ def get_backup_directories() -> dict[str, str]: "webchat": get_astrbot_webchat_path(), # WebChat 数据 "temp": get_astrbot_temp_path(), # 临时文件 "skills": get_astrbot_skills_path(), # Skills - "workspaces": get_astrbot_workspaces_path(), # Session and project files } diff --git a/docs/en/use/webui.md b/docs/en/use/webui.md index e65e4664f4f..32ab6ea70ca 100644 --- a/docs/en/use/webui.md +++ b/docs/en/use/webui.md @@ -135,7 +135,7 @@ The page title and section navigation stay fixed while only the settings pane on System configuration changes save automatically. Check for a successful save message and restart AstrBot if the page indicates that a restart is required. -Backups under `Maintenance` include session aliases, model usage statistics, OpenAPI access keys, MCP server configuration, and session and project files under `data/workspaces/`. Custom workspace directories outside `data/workspaces/` require a separate backup. TOTP trusted devices are excluded; replacing data from a backup clears existing device trust and requires device verification again. Older backups cannot recover data they did not include: export a new backup from the original instance before migrating. If a backup has no MCP configuration, the target instance's MCP configuration is kept. +Backups under `Maintenance` include session aliases, model usage statistics, OpenAPI access keys, and MCP server configuration. Workspace files require a separate backup. TOTP trusted devices are excluded; replacing data from a backup clears existing device trust and requires device verification again. Older backups cannot recover data they did not include: export a new backup from the original instance before migrating. If a backup has no MCP configuration, the target instance's MCP configuration is kept. ## Plugins diff --git a/docs/zh/use/webui.md b/docs/zh/use/webui.md index b84b448dd12..57422f77224 100644 --- a/docs/zh/use/webui.md +++ b/docs/zh/use/webui.md @@ -135,7 +135,7 @@ ChatUI 支持以下常用能力： 系统配置修改后自动保存，请确认保存成功提示；如果页面提示需要重启，再按提示重启 AstrBot。 -`维护` 中的备份包含会话别名、模型用量统计、OpenAPI 访问密钥、MCP 服务配置，以及 `data/workspaces/` 下的会话和项目文件。位于 `data/workspaces/` 之外的自定义工作区需要单独备份。TOTP 受信任设备记录不纳入备份；替换恢复时会清除现有设备信任，需要重新验证设备。旧备份无法恢复未包含的数据，迁移前请从原实例重新导出备份。如果备份中没有 MCP 配置，则保留目标实例现有的 MCP 配置。 +`维护` 中的备份包含会话别名、模型用量统计、OpenAPI 访问密钥和 MCP 服务配置。工作区文件需要单独备份。TOTP 受信任设备记录不纳入备份；替换恢复时会清除现有设备信任，需要重新验证设备。旧备份无法恢复未包含的数据，迁移前请从原实例重新导出备份。如果备份中没有 MCP 配置，则保留目标实例现有的 MCP 配置。 ## 插件 diff --git a/tests/test_backup.py b/tests/test_backup.py index 31d0ecf8ec9..25ccb2fca16 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -1220,16 +1220,16 @@ async def test_device_trust_is_excluded_and_cleared(self, tmp_path, monkeypatch) @pytest.mark.parametrize( "existing_target", [True, False], ids=["existing", "fresh"] ) - async def test_mcp_and_workspaces_roundtrip( + async def test_mcp_config_roundtrip( self, tmp_path, monkeypatch, include_mcp, existing_target ): - """Restore configuration and workspace files across distinct AstrBot roots. + """Restore MCP configuration across distinct AstrBot roots. Args: tmp_path: Temporary directory for both instances and their databases. monkeypatch: Fixture used to switch between instance roots. include_mcp: Whether the backup contains an MCP configuration. - existing_target: Whether the target already has configuration and files. + existing_target: Whether the target already has MCP configuration. """ source_root = tmp_path / "source" target_root = tmp_path / "target" @@ -1245,20 +1245,10 @@ async def test_mcp_and_workspaces_roundtrip( ).encode("utf-8") if include_mcp: (source_data / "mcp_server.json").write_bytes(mcp_content) - workspace_files = { - "project_demo/report.txt": b"Project report", - "session/skills/example/SKILL.md": b"# Workspace skill", - "session/output.bin": bytes(range(256)), - } - for relative_path, content in workspace_files.items(): - file_path = source_data / "workspaces" / relative_path - file_path.parent.mkdir(parents=True, exist_ok=True) - file_path.write_bytes(content) target_data = target_root / "data" target_mcp = target_data / "mcp_server.json" if existing_target: - (target_data / "workspaces").mkdir(parents=True) - (target_data / "workspaces" / "stale.txt").write_text("Old workspace") + target_data.mkdir(parents=True) target_mcp.write_bytes(b'{"mcpServers": {}}') source = SQLiteDatabase(str(tmp_path / "source.db")) @@ -1271,7 +1261,6 @@ async def test_mcp_and_workspaces_roundtrip( zip_path = await exporter.export_all(output_dir=str(tmp_path / "backups")) with zipfile.ZipFile(zip_path) as archive: manifest = json.loads(archive.read("manifest.json")) - assert "workspaces" in manifest["directories"] assert ("config/mcp_server.json" in archive.namelist()) == include_mcp if include_mcp: assert manifest["checksums"]["config/mcp_server.json"] == ( @@ -1284,12 +1273,6 @@ async def test_mcp_and_workspaces_roundtrip( ).import_all(zip_path) assert result.success, result.errors assert not result.warnings - assert result.imported_directories["workspaces"] == len(workspace_files) - for relative_path, content in workspace_files.items(): - assert ( - target_data / "workspaces" / relative_path - ).read_bytes() == content - assert not (target_data / "workspaces" / "stale.txt").exists() if include_mcp: assert target_mcp.read_bytes() == mcp_content assert result.imported_files["mcp_config"] == 1