You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
39 lines
1.0 KiB
Python
39 lines
1.0 KiB
Python
import asyncio
|
|
import sys
|
|
import base as _
|
|
import utils.local as local
|
|
from service.database import DatabaseService
|
|
|
|
from service.embedding_search import EmbeddingSearchService
|
|
from service.mediawiki_api import MediaWikiApi
|
|
|
|
async def main():
|
|
dbs = await DatabaseService.create()
|
|
mw_api = MediaWikiApi.create()
|
|
|
|
start_title = sys.argv[1] if len(sys.argv) > 1 else None
|
|
|
|
continue_key = None
|
|
while True:
|
|
page_res = await mw_api.get_all_pages(continue_key, start_title)
|
|
start_title = None
|
|
|
|
title_list = page_res["title_list"]
|
|
|
|
for page_title in title_list:
|
|
print("Indexing %s" % page_title)
|
|
async with EmbeddingSearchService(dbs, page_title) as embedding_search:
|
|
await embedding_search.update_title_index(True)
|
|
|
|
if not page_res["continue_key"]:
|
|
break
|
|
|
|
continue_key = page_res["continue_key"]
|
|
|
|
print("Done")
|
|
|
|
await local.noawait.end()
|
|
await asyncio.sleep(1)
|
|
|
|
if __name__ == '__main__':
|
|
local.loop.run_until_complete(main()) |