From a6e80850cce7bac9c2601915c05d543eda837c3d Mon Sep 17 00:00:00 2001 From: Lex Lim Date: Tue, 7 May 2024 11:42:13 +0000 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=8A=A8=E6=80=81=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E7=9A=84=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/route.py | 41 ---------- init/server/base_api.py | 9 +++ init/server/chat_complete.py | 22 ++++++ init/server/embedding_search.py | 17 ++++ init/server/toolbox_ui.py | 16 ++++ init/service/database.py | 23 ++++++ init/service/mw_api.py | 33 ++++++++ init/service/tiktoken.py | 16 ++++ main.py | 77 +++++-------------- {api => server}/controller/ChatComplete.py | 10 +-- {api => server}/controller/EmbeddingSearch.py | 4 +- {api => server}/controller/Hanja.py | 0 {api => server}/controller/Hanzi.py | 0 {api => server}/controller/Kanji.py | 0 .../controller/ToolboxIndex.py | 6 +- .../controller/task/ChatCompleteTask.py | 0 {api => server}/model/base.py | 0 .../model/chat_complete/bot_persona.py | 4 +- .../chat_complete/bot_persona_category.py | 2 +- .../model/chat_complete/conversation.py | 4 +- .../model/embedding_search/page_index.py | 2 +- .../embedding_search/title_collection.py | 2 +- .../model/embedding_search/title_index.py | 2 +- .../model/toolbox_ui}/conversation.py | 8 +- .../model/toolbox_ui}/page_title.py | 4 +- server/route/base_api.py | 15 ++++ server/route/chat_complete.py | 15 ++++ server/route/embedding_search.py | 10 +++ server/route/toolbox_ui.py | 16 ++++ service/chat_complete.py | 6 +- service/embedding_search.py | 6 +- test/create_token.py | 2 +- test/title_index.py | 2 +- utils/server.py | 23 ++++++ 34 files changed, 266 insertions(+), 131 deletions(-) delete mode 100644 api/route.py create mode 100644 init/server/base_api.py create mode 100644 init/server/chat_complete.py create mode 100644 init/server/embedding_search.py create mode 100644 init/server/toolbox_ui.py create mode 100644 init/service/database.py create mode 100644 init/service/mw_api.py create mode 100644 init/service/tiktoken.py rename {api => server}/controller/ChatComplete.py (98%) rename {api => server}/controller/EmbeddingSearch.py (98%) rename {api => server}/controller/Hanja.py (100%) rename {api => server}/controller/Hanzi.py (100%) rename {api => server}/controller/Kanji.py (100%) rename api/controller/Index.py => server/controller/ToolboxIndex.py (98%) rename {api => server}/controller/task/ChatCompleteTask.py (100%) rename {api => server}/model/base.py (100%) rename {api => server}/model/chat_complete/bot_persona.py (96%) rename {api => server}/model/chat_complete/bot_persona_category.py (94%) rename {api => server}/model/chat_complete/conversation.py (96%) rename {api => server}/model/embedding_search/page_index.py (99%) rename {api => server}/model/embedding_search/title_collection.py (97%) rename {api => server}/model/embedding_search/title_index.py (98%) rename {api/model/toolkit_ui => server/model/toolbox_ui}/conversation.py (93%) rename {api/model/toolkit_ui => server/model/toolbox_ui}/page_title.py (95%) create mode 100644 server/route/base_api.py create mode 100644 server/route/chat_complete.py create mode 100644 server/route/embedding_search.py create mode 100644 server/route/toolbox_ui.py create mode 100644 utils/server.py diff --git a/api/route.py b/api/route.py deleted file mode 100644 index a87fa1a..0000000 --- a/api/route.py +++ /dev/null @@ -1,41 +0,0 @@ -from aiohttp import web -from api.controller.ChatComplete import ChatComplete - -from api.controller.Hanzi import Hanzi -from api.controller.Index import Index -from api.controller.Kanji import Kanji -from api.controller.Hanja import Hanja -from api.controller.EmbeddingSearch import EmbeddingSearch - -def init(app: web.Application): - app.router.add_routes([ - web.route('*', '/hanzi/pinyin/', Hanzi.hanziToPinyin), - web.route('*', '/hanzi/split/', Hanzi.splitHanzi), - - web.route('*', '/kanji/romaji/', Kanji.kanji2romaji), - - web.route('*', '/hanja/romaja/', Hanja.hanja2roma), - - web.route('*', '/title/info', Index.update_title_info), - web.route('*', '/title/search', Index.search_title), - web.route('*', '/user/info', Index.get_user_info), - web.route('*', '/conversation/list', Index.get_conversation_list), - web.route('*', '/conversation/info', Index.get_conversation_info), - web.route('POST', '/conversation/remove', Index.remove_conversation), - web.route('DELETE', '/conversation/remove', Index.remove_conversation), - web.route('POST', '/conversation/set_pinned', Index.set_conversation_pinned), - web.route('POST', '/conversation/set_title', Index.set_conversation_title), - - web.route('*', '/embedding_search/index_page', EmbeddingSearch.index_page), - web.route('*', '/embedding_search/search', EmbeddingSearch.search), - web.route('POST', '/sys/embedding_search/title/update', EmbeddingSearch.sys_update_title_info), - - web.route('*', '/chatcomplete/conversation_chunk/list', ChatComplete.get_conversation_chunk_list), - web.route('*', '/chatcomplete/conversation_chunk/info', ChatComplete.get_conversation_chunk), - web.route('POST', '/chatcomplete/conversation/fork', ChatComplete.fork_conversation), - web.route('POST', '/chatcomplete/message', ChatComplete.start_chat_complete), - web.route('GET', '/chatcomplete/message/stream', ChatComplete.chat_complete_stream), - web.route('POST', '/chatcomplete/get_point_cost', ChatComplete.get_point_cost), - web.route('*', '/chatcomplete/persona/list', ChatComplete.get_persona_list), - web.route('*', '/chatcomplete/persona/info', ChatComplete.get_persona_info), - ]) diff --git a/init/server/base_api.py b/init/server/base_api.py new file mode 100644 index 0000000..a032456 --- /dev/null +++ b/init/server/base_api.py @@ -0,0 +1,9 @@ +from aiohttp import web +from utils.server import register_server_module + +from server.route.base_api import register_route + +def init(app: web.Application): + register_route(app) + +register_server_module("base_api", init) \ No newline at end of file diff --git a/init/server/chat_complete.py b/init/server/chat_complete.py new file mode 100644 index 0000000..85c98bf --- /dev/null +++ b/init/server/chat_complete.py @@ -0,0 +1,22 @@ +from aiohttp import web +from utils.server import register_server_module + +import init.service.mw_api as _ # Init mediawiki api +import init.service.database as _ # Init database +import init.service.tiktoken as _ # Init tiktoken + +import toolbox_ui as _ # Init toolbox ui +import embedding_search as _ # Init embedding search + +# Auto create database tables +from server.model.chat_complete.conversation import ConversationChunkModel as _ +from server.model.chat_complete.bot_persona_category import BotPersonaCategoryModel as _ +from server.model.chat_complete.bot_persona import BotPersonaModel as _ + +# Route +from server.route.chat_complete import register_route + +def init(app: web.Application): + register_route(app) + +register_server_module("chat_complete", init) \ No newline at end of file diff --git a/init/server/embedding_search.py b/init/server/embedding_search.py new file mode 100644 index 0000000..810a923 --- /dev/null +++ b/init/server/embedding_search.py @@ -0,0 +1,17 @@ +from aiohttp import web +from utils.server import register_server_module + +import init.service.database as _ # Init database +import init.service.tiktoken as _ # Init tiktoken + +# Auto create database tables +from server.model.embedding_search.title_collection import TitleCollectionModel as _ +from server.model.embedding_search.title_index import TitleIndexModel as _ + +# Route +from server.route.embedding_search import register_route + +def init(app: web.Application): + register_route(app) + +register_server_module("embedding_search", init) \ No newline at end of file diff --git a/init/server/toolbox_ui.py b/init/server/toolbox_ui.py new file mode 100644 index 0000000..5d0e80a --- /dev/null +++ b/init/server/toolbox_ui.py @@ -0,0 +1,16 @@ +from aiohttp import web +from utils.server import register_server_module + +import init.service.database as _ # Init database + +# Auto create database tables +from server.model.toolbox_ui.page_title import PageTitleModel as _ +from server.model.toolbox_ui.conversation import ConversationModel as _ + +# Route +from server.route.toolbox_ui import register_route + +def init(app: web.Application): + register_route(app) + +register_server_module("toolbox_ui", init) \ No newline at end of file diff --git a/init/service/database.py b/init/service/database.py new file mode 100644 index 0000000..32e3fd0 --- /dev/null +++ b/init/service/database.py @@ -0,0 +1,23 @@ +from __future__ import annotations +from aiohttp import web +from utils.server import register_server_module + +from server.model.base import BaseModel +from service.database import DatabaseService + +async def init_database(app: web.Application): + dbs = await DatabaseService.create(app) + print("Database connected.") + + async with dbs.engine.begin() as conn: + await conn.run_sync(BaseModel.metadata.create_all) + +async def close_database(app: web.Application): + dbs = await DatabaseService.create(app) + await dbs.close() + +def init(app: web.Application): + app.on_startup.append(init_database) + app.on_cleanup.append(close_database) + +register_server_module("database", init) \ No newline at end of file diff --git a/init/service/mw_api.py b/init/service/mw_api.py new file mode 100644 index 0000000..b2c28af --- /dev/null +++ b/init/service/mw_api.py @@ -0,0 +1,33 @@ +from __future__ import annotations +from aiohttp import web +from utils.server import register_server_module + +from lib.config import Config +from service.mediawiki_api import MediaWikiApi + + +async def init_mw_api(app: web.Application): + mw_api = MediaWikiApi.create() + + bot_username = Config.get("mediawiki.bot_username", "", str) + bot_password = Config.get("mediawiki.bot_password", "", str) + + if bot_username and bot_password: + try: + await mw_api.robot_login(bot_username, bot_password) + except Exception as e: + print("Cannot login to Robot account, please check config.") + + site_meta = await mw_api.get_site_meta() + + print( + "Connected to Wiki %s, Robot username: %s" + % (site_meta["sitename"], site_meta["user"]) + ) + + +def init(app: web.Application): + app.on_startup.append(init_mw_api) + + +register_server_module("mediawiki_api", init) diff --git a/init/service/tiktoken.py b/init/service/tiktoken.py new file mode 100644 index 0000000..780712c --- /dev/null +++ b/init/service/tiktoken.py @@ -0,0 +1,16 @@ +from aiohttp import web +from utils.server import register_server_module + +from service.tiktoken import TikTokenService + + +async def init_tiktoken(app: web.Application): + await TikTokenService.create() + print("Tiktoken model loaded.") + + +async def init(app: web.Application): + app.on_startup.append(init_tiktoken) + + +register_server_module("tiktoken", init) diff --git a/main.py b/main.py index dfff894..7a078e0 100644 --- a/main.py +++ b/main.py @@ -1,29 +1,17 @@ import sys import traceback from lib.config import Config +from utils.server import get_server_modules Config.load_config("config.toml") from utils.local import loop, noawait from aiohttp import web import utils.local as local -import api.route +import server.route as api_route import utils.web -from service.database import DatabaseService from service.mediawiki_api import MediaWikiApi -# Auto create Table -from api.model.base import BaseModel -from api.model.toolkit_ui.page_title import PageTitleModel as _ -from api.model.toolkit_ui.conversation import ConversationModel as _ -from api.model.chat_complete.conversation import ConversationChunkModel as _ -from api.model.chat_complete.bot_persona_category import BotPersonaCategoryModel as _ -from api.model.chat_complete.bot_persona import BotPersonaModel as _ -from api.model.embedding_search.title_collection import TitleCollectionModel as _ -from api.model.embedding_search.title_index import TitleIndexModel as _ - -from service.tiktoken import TikTokenService - async def index(request: web.Request): return await utils.web.api_response( @@ -65,42 +53,6 @@ async def error_handler(request, handler): ) -async def init_mw_api(app: web.Application): - mw_api = MediaWikiApi.create() - - bot_username = Config.get("mediawiki.bot_username", "", str) - bot_password = Config.get("mediawiki.bot_password", "", str) - - if bot_username and bot_password: - try: - await mw_api.robot_login(bot_username, bot_password) - except Exception as e: - print("Cannot login to Robot account, please check config.") - - site_meta = await mw_api.get_site_meta() - - print( - "Connected to Wiki %s, Robot username: %s" - % (site_meta["sitename"], site_meta["user"]) - ) - - -async def init_database(app: web.Application): - dbs = await DatabaseService.create(app) - print("Database connected.") - - async with dbs.engine.begin() as conn: - await conn.run_sync(BaseModel.metadata.create_all) - -async def close_database(app: web.Application): - dbs = await DatabaseService.create(app) - await dbs.close() - -async def init_tiktoken(app: web.Application): - await TikTokenService.create() - print("Tiktoken model loaded.") - - async def stop_noawait_pool(app: web.Application): await noawait.end() @@ -113,20 +65,29 @@ if __name__ == "__main__": ] ) - if Config.get("database.host"): - app.on_startup.append(init_database) - app.on_cleanup.append(close_database) + import init.server.base_api as _ + + # Dynamically import modules + if Config.get("toolbox_ui.enable"): + import init.server.toolbox_ui as _ + + if Config.get("embedding.enable"): + import init.server.embedding_search as _ + + if Config.get("chatcomplete.enable"): + import init.server.chat_complete as _ - if Config.get("mediawiki.api_endpoint"): - app.on_startup.append(init_mw_api) + # Initialize server modules + server_modules = get_server_modules() + print(server_modules) - if Config.get("chatcomplete.enabled"): - app.on_startup.append(init_tiktoken) + for server_module in server_modules: + server_module["init"](app) app.on_shutdown.append(stop_noawait_pool) app.router.add_route("*", "/", index) - api.route.init(app) + api_route.init(app) server_port = Config.get("port", 8144, int) diff --git a/api/controller/ChatComplete.py b/server/controller/ChatComplete.py similarity index 98% rename from api/controller/ChatComplete.py rename to server/controller/ChatComplete.py index e079861..2a21f4a 100644 --- a/api/controller/ChatComplete.py +++ b/server/controller/ChatComplete.py @@ -2,13 +2,13 @@ from __future__ import annotations import asyncio import time import traceback -from api.controller.task.ChatCompleteTask import ChatCompleteTask -from api.model.base import clone_model -from api.model.chat_complete.bot_persona import BotPersonaHelper -from api.model.toolkit_ui.conversation import ConversationHelper +from server.controller.task.ChatCompleteTask import ChatCompleteTask +from server.model.base import clone_model +from server.model.chat_complete.bot_persona import BotPersonaHelper +from server.model.toolbox_ui.conversation import ConversationHelper from utils.local import noawait from aiohttp import web -from api.model.chat_complete.conversation import ConversationChunkHelper, ConversationModel, ConversationChunkModel +from server.model.chat_complete.conversation import ConversationChunkHelper, ConversationModel, ConversationChunkModel from service.chat_complete import ChatCompleteQuestionTooLongException, ChatCompleteServiceResponse from service.database import DatabaseService from service.mediawiki_api import MediaWikiApi, MediaWikiPageNotFoundException, MediaWikiUserNoEnoughPointsException diff --git a/api/controller/EmbeddingSearch.py b/server/controller/EmbeddingSearch.py similarity index 98% rename from api/controller/EmbeddingSearch.py rename to server/controller/EmbeddingSearch.py index ee9db3f..6b3f976 100644 --- a/api/controller/EmbeddingSearch.py +++ b/server/controller/EmbeddingSearch.py @@ -1,8 +1,8 @@ import sys import traceback from aiohttp import web -from api.model.embedding_search.title_collection import TitleCollectionHelper -from api.model.embedding_search.title_index import TitleIndexHelper +from server.model.embedding_search.title_collection import TitleCollectionHelper +from server.model.embedding_search.title_index import TitleIndexHelper from service.database import DatabaseService from service.embedding_search import EmbeddingRunningException, EmbeddingSearchService from service.mediawiki_api import MediaWikiApi, MediaWikiApiException, MediaWikiPageNotFoundException diff --git a/api/controller/Hanja.py b/server/controller/Hanja.py similarity index 100% rename from api/controller/Hanja.py rename to server/controller/Hanja.py diff --git a/api/controller/Hanzi.py b/server/controller/Hanzi.py similarity index 100% rename from api/controller/Hanzi.py rename to server/controller/Hanzi.py diff --git a/api/controller/Kanji.py b/server/controller/Kanji.py similarity index 100% rename from api/controller/Kanji.py rename to server/controller/Kanji.py diff --git a/api/controller/Index.py b/server/controller/ToolboxIndex.py similarity index 98% rename from api/controller/Index.py rename to server/controller/ToolboxIndex.py index 7f0051f..426c5f6 100644 --- a/api/controller/Index.py +++ b/server/controller/ToolboxIndex.py @@ -2,15 +2,15 @@ import sys import time import traceback from aiohttp import web -from api.model.toolkit_ui.conversation import ConversationHelper -from api.model.toolkit_ui.page_title import PageTitleHelper +from server.model.toolbox_ui.conversation import ConversationHelper +from server.model.toolbox_ui.page_title import PageTitleHelper from service.database import DatabaseService from service.event import EventService from service.mediawiki_api import MediaWikiApi, MediaWikiApiException, MediaWikiPageNotFoundException import utils.web -class Index: +class ToolboxIndex: @staticmethod @utils.web.token_auth async def update_title_info(request: web.Request): diff --git a/api/controller/task/ChatCompleteTask.py b/server/controller/task/ChatCompleteTask.py similarity index 100% rename from api/controller/task/ChatCompleteTask.py rename to server/controller/task/ChatCompleteTask.py diff --git a/api/model/base.py b/server/model/base.py similarity index 100% rename from api/model/base.py rename to server/model/base.py diff --git a/api/model/chat_complete/bot_persona.py b/server/model/chat_complete/bot_persona.py similarity index 96% rename from api/model/chat_complete/bot_persona.py rename to server/model/chat_complete/bot_persona.py index 7d34670..24b1d24 100644 --- a/api/model/chat_complete/bot_persona.py +++ b/server/model/chat_complete/bot_persona.py @@ -2,13 +2,13 @@ from __future__ import annotations import math from typing import Optional import sqlalchemy -from api.model.base import BaseHelper, BaseModel +from server.model.base import BaseHelper, BaseModel import sqlalchemy from sqlalchemy import select, update from sqlalchemy.orm import mapped_column, relationship, load_only, Mapped -from api.model.chat_complete.bot_persona_category import BotPersonaCategoryModel +from server.model.chat_complete.bot_persona_category import BotPersonaCategoryModel from service.database import DatabaseService diff --git a/api/model/chat_complete/bot_persona_category.py b/server/model/chat_complete/bot_persona_category.py similarity index 94% rename from api/model/chat_complete/bot_persona_category.py rename to server/model/chat_complete/bot_persona_category.py index 3bbdc9f..a541dea 100644 --- a/api/model/chat_complete/bot_persona_category.py +++ b/server/model/chat_complete/bot_persona_category.py @@ -1,6 +1,6 @@ from __future__ import annotations import sqlalchemy -from api.model.base import BaseHelper, BaseModel +from server.model.base import BaseHelper, BaseModel import sqlalchemy from sqlalchemy import select, update diff --git a/api/model/chat_complete/conversation.py b/server/model/chat_complete/conversation.py similarity index 96% rename from api/model/chat_complete/conversation.py rename to server/model/chat_complete/conversation.py index f29096a..e9d981f 100644 --- a/api/model/chat_complete/conversation.py +++ b/server/model/chat_complete/conversation.py @@ -5,8 +5,8 @@ import sqlalchemy from sqlalchemy import select, update from sqlalchemy.orm import mapped_column, relationship, Mapped -from api.model.base import BaseHelper, BaseModel -from api.model.toolkit_ui.conversation import ConversationModel +from server.model.base import BaseHelper, BaseModel +from server.model.toolbox_ui.conversation import ConversationModel from service.database import DatabaseService from service.event import EventService diff --git a/api/model/embedding_search/page_index.py b/server/model/embedding_search/page_index.py similarity index 99% rename from api/model/embedding_search/page_index.py rename to server/model/embedding_search/page_index.py index 2baaeee..a9c9e29 100644 --- a/api/model/embedding_search/page_index.py +++ b/server/model/embedding_search/page_index.py @@ -3,7 +3,7 @@ import hashlib from typing import Optional, Type import asyncpg -from api.model.base import BaseModel +from server.model.base import BaseModel import numpy as np import sqlalchemy from lib.config import Config diff --git a/api/model/embedding_search/title_collection.py b/server/model/embedding_search/title_collection.py similarity index 97% rename from api/model/embedding_search/title_collection.py rename to server/model/embedding_search/title_collection.py index cd928e7..f016f38 100644 --- a/api/model/embedding_search/title_collection.py +++ b/server/model/embedding_search/title_collection.py @@ -3,7 +3,7 @@ import sqlalchemy from sqlalchemy import select, update, delete from sqlalchemy.orm import mapped_column, Mapped -from api.model.base import BaseHelper, BaseModel +from server.model.base import BaseHelper, BaseModel from service.database import DatabaseService class TitleCollectionModel(BaseModel): diff --git a/api/model/embedding_search/title_index.py b/server/model/embedding_search/title_index.py similarity index 98% rename from api/model/embedding_search/title_index.py rename to server/model/embedding_search/title_index.py index 2ecad92..405db21 100644 --- a/api/model/embedding_search/title_index.py +++ b/server/model/embedding_search/title_index.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import mapped_column, relationship, Mapped, deferred, defer from sqlalchemy.ext.asyncio import AsyncEngine from lib.config import Config -from api.model.base import BaseHelper, BaseModel +from server.model.base import BaseHelper, BaseModel from service.database import DatabaseService embedding_vector_size = Config.get("embedding.vector_size", 1536, int) diff --git a/api/model/toolkit_ui/conversation.py b/server/model/toolbox_ui/conversation.py similarity index 93% rename from api/model/toolkit_ui/conversation.py rename to server/model/toolbox_ui/conversation.py index 8da1c2b..8555a30 100644 --- a/api/model/toolkit_ui/conversation.py +++ b/server/model/toolbox_ui/conversation.py @@ -6,13 +6,13 @@ import sqlalchemy from sqlalchemy import update from sqlalchemy.orm import mapped_column, relationship, Mapped -from api.model.base import BaseHelper, BaseModel -from api.model.toolkit_ui.page_title import PageTitleModel +from server.model.base import BaseHelper, BaseModel +from server.model.toolbox_ui.page_title import PageTitleModel from service.database import DatabaseService class ConversationModel(BaseModel): - __tablename__ = "toolkit_ui_conversation" + __tablename__ = "toolbox_ui_conversation" id: Mapped[int] = mapped_column(sqlalchemy.Integer, primary_key=True, autoincrement=True) user_id: Mapped[int] = mapped_column(sqlalchemy.Integer, index=True) @@ -20,7 +20,7 @@ class ConversationModel(BaseModel): title: Mapped[str] = mapped_column(sqlalchemy.String(255), nullable=True) thumbnail: Mapped[str] = mapped_column(sqlalchemy.Text(), nullable=True) description: Mapped[str] = mapped_column(sqlalchemy.Text(), nullable=True) - page_id: Mapped[int] = mapped_column(sqlalchemy.ForeignKey("toolkit_ui_page_title.page_id"), index=True, nullable=True) + page_id: Mapped[int] = mapped_column(sqlalchemy.ForeignKey("toolbox_ui_page_title.page_id"), index=True, nullable=True) rev_id: Mapped[int] = mapped_column(sqlalchemy.Integer, nullable=True) updated_at: Mapped[int] = mapped_column(sqlalchemy.BigInteger, index=True) pinned: Mapped[bool] = mapped_column( diff --git a/api/model/toolkit_ui/page_title.py b/server/model/toolbox_ui/page_title.py similarity index 95% rename from api/model/toolkit_ui/page_title.py rename to server/model/toolbox_ui/page_title.py index cedb811..cd474e1 100644 --- a/api/model/toolkit_ui/page_title.py +++ b/server/model/toolbox_ui/page_title.py @@ -6,12 +6,12 @@ import sqlalchemy from sqlalchemy import select, update from sqlalchemy.orm import mapped_column, Mapped -from api.model.base import BaseHelper, BaseModel +from server.model.base import BaseHelper, BaseModel from service.database import DatabaseService class PageTitleModel(BaseModel): - __tablename__ = "toolkit_ui_page_title" + __tablename__ = "toolbox_ui_page_title" id: Mapped[int] = mapped_column( sqlalchemy.Integer, primary_key=True, autoincrement=True) diff --git a/server/route/base_api.py b/server/route/base_api.py new file mode 100644 index 0000000..4d5179f --- /dev/null +++ b/server/route/base_api.py @@ -0,0 +1,15 @@ +from aiohttp import web + +from server.controller.Hanja import Hanja +from server.controller.Hanzi import Hanzi +from server.controller.Kanji import Kanji + +def register_route(app: web.Application): + app.router.add_routes([ + web.route('*', '/hanzi/pinyin/', Hanzi.hanziToPinyin), + web.route('*', '/hanzi/split/', Hanzi.splitHanzi), + + web.route('*', '/kanji/romaji/', Kanji.kanji2romaji), + + web.route('*', '/hanja/romaja/', Hanja.hanja2roma), + ]) \ No newline at end of file diff --git a/server/route/chat_complete.py b/server/route/chat_complete.py new file mode 100644 index 0000000..0c74881 --- /dev/null +++ b/server/route/chat_complete.py @@ -0,0 +1,15 @@ +from aiohttp import web + +from server.controller.ChatComplete import ChatComplete + +def register_route(app: web.Application): + app.router.add_routes([ + web.route('*', '/chatcomplete/conversation_chunk/list', ChatComplete.get_conversation_chunk_list), + web.route('*', '/chatcomplete/conversation_chunk/info', ChatComplete.get_conversation_chunk), + web.route('POST', '/chatcomplete/conversation/fork', ChatComplete.fork_conversation), + web.route('POST', '/chatcomplete/message', ChatComplete.start_chat_complete), + web.route('GET', '/chatcomplete/message/stream', ChatComplete.chat_complete_stream), + web.route('POST', '/chatcomplete/get_point_cost', ChatComplete.get_point_cost), + web.route('*', '/chatcomplete/persona/list', ChatComplete.get_persona_list), + web.route('*', '/chatcomplete/persona/info', ChatComplete.get_persona_info), + ]) \ No newline at end of file diff --git a/server/route/embedding_search.py b/server/route/embedding_search.py new file mode 100644 index 0000000..74afebb --- /dev/null +++ b/server/route/embedding_search.py @@ -0,0 +1,10 @@ +from aiohttp import web + +from server.controller.EmbeddingSearch import EmbeddingSearch + +def register_route(app: web.Application): + app.router.add_routes([ + web.route('*', '/embedding_search/index_page', EmbeddingSearch.index_page), + web.route('*', '/embedding_search/search', EmbeddingSearch.search), + web.route('POST', '/sys/embedding_search/title/update', EmbeddingSearch.sys_update_title_info), + ]) \ No newline at end of file diff --git a/server/route/toolbox_ui.py b/server/route/toolbox_ui.py new file mode 100644 index 0000000..f16eef3 --- /dev/null +++ b/server/route/toolbox_ui.py @@ -0,0 +1,16 @@ +from aiohttp import web + +from server.controller.ToolboxIndex import ToolboxIndex + +def register_route(app: web.Application): + app.router.add_routes([ + web.route('*', '/title/info', ToolboxIndex.update_title_info), + web.route('*', '/title/search', ToolboxIndex.search_title), + web.route('*', '/user/info', ToolboxIndex.get_user_info), + web.route('*', '/conversation/list', ToolboxIndex.get_conversation_list), + web.route('*', '/conversation/info', ToolboxIndex.get_conversation_info), + web.route('POST', '/conversation/remove', ToolboxIndex.remove_conversation), + web.route('DELETE', '/conversation/remove', ToolboxIndex.remove_conversation), + web.route('POST', '/conversation/set_pinned', ToolboxIndex.set_conversation_pinned), + web.route('POST', '/conversation/set_title', ToolboxIndex.set_conversation_title), + ]) \ No newline at end of file diff --git a/service/chat_complete.py b/service/chat_complete.py index 1726e7c..1c573b7 100644 --- a/service/chat_complete.py +++ b/service/chat_complete.py @@ -3,13 +3,13 @@ import time import traceback from typing import Optional, Tuple, TypedDict -from api.model.chat_complete.bot_persona import BotPersonaHelper -from api.model.chat_complete.conversation import ( +from server.model.chat_complete.bot_persona import BotPersonaHelper +from server.model.chat_complete.conversation import ( ConversationChunkHelper, ConversationChunkModel, ) import sys -from api.model.toolkit_ui.conversation import ConversationHelper, ConversationModel +from server.model.toolbox_ui.conversation import ConversationHelper, ConversationModel from lib.config import Config import utils.config, utils.web diff --git a/service/embedding_search.py b/service/embedding_search.py index 7f4a52a..aea6193 100644 --- a/service/embedding_search.py +++ b/service/embedding_search.py @@ -2,12 +2,12 @@ from __future__ import annotations from typing import Optional, TypedDict import sqlalchemy -from api.model.embedding_search.title_collection import ( +from server.model.embedding_search.title_collection import ( TitleCollectionHelper, TitleCollectionModel, ) -from api.model.embedding_search.title_index import TitleIndexHelper, TitleIndexModel -from api.model.embedding_search.page_index import PageIndexHelper +from server.model.embedding_search.title_index import TitleIndexHelper, TitleIndexModel +from server.model.embedding_search.page_index import PageIndexHelper from service.database import DatabaseService from service.mediawiki_api import MediaWikiApi from service.openai_api import OpenAIApi diff --git a/test/create_token.py b/test/create_token.py index 345bd5e..07e34c9 100644 --- a/test/create_token.py +++ b/test/create_token.py @@ -2,7 +2,7 @@ import asyncio import base from sqlalchemy import select -from api.model.embedding_search.title_index import TitleIndexModel +from server.model.embedding_search.title_index import TitleIndexModel import utils.local as local from service.database import DatabaseService diff --git a/test/title_index.py b/test/title_index.py index 0a5a127..f067d99 100644 --- a/test/title_index.py +++ b/test/title_index.py @@ -2,7 +2,7 @@ import asyncio import base from sqlalchemy import select -from api.model.embedding_search.title_index import TitleIndexModel +from server.model.embedding_search.title_index import TitleIndexModel import utils.local as local from service.database import DatabaseService diff --git a/utils/server.py b/utils/server.py new file mode 100644 index 0000000..e0d564a --- /dev/null +++ b/utils/server.py @@ -0,0 +1,23 @@ +from __future__ import annotations +from typing import TypedDict + +class ServerModuleItem(TypedDict): + name: str + init: callable + +_SERVER_MODULES: list[ServerModuleItem] = [] + +def register_server_module(name: str, init_method: callable): + global _SERVER_MODULES + + for module in _SERVER_MODULES: + if module["name"] == name: + return + + _SERVER_MODULES.append({ + "name": name, + "init": init_method + }) + +def get_server_modules() -> list[ServerModuleItem]: + return _SERVER_MODULES \ No newline at end of file