You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
27 lines
784 B
Python
27 lines
784 B
Python
2 years ago
|
import asyncio
|
||
|
|
import time
|
||
|
|
import base
|
||
|
|
from local import loop, noawait
|
||
|
|
from service.bert_embedding import bert_embedding_queue
|
||
|
|
|
||
|
|
async def main():
|
||
|
|
embedding_list = []
|
||
|
|
start_time = time.time()
|
||
|
|
queue = []
|
||
|
|
with open("test/test.md", "r", encoding="utf-8") as fp:
|
||
|
|
text = fp.read()
|
||
|
|
lines = text.split("\n")
|
||
|
|
for line in lines:
|
||
|
|
line = line.strip()
|
||
|
|
if line == "":
|
||
|
|
continue
|
||
|
|
|
||
|
|
queue.append(bert_embedding_queue.get_embeddings(line))
|
||
|
|
embedding_list = await asyncio.gather(*queue)
|
||
|
|
end_time = time.time()
|
||
|
|
print("time cost: %.4f" % (end_time - start_time))
|
||
|
|
print("dimensions: %d" % len(embedding_list[0]))
|
||
|
|
await noawait.end()
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
loop.run_until_complete(main())
|