|
|
import re
|
|
|
from bs4 import BeautifulSoup
|
|
|
from markdownify import MarkdownConverter
|
|
|
|
|
|
def stripMarkdown(text):
|
|
|
# Remove headers
|
|
|
text = re.sub(r'#+(\s+)', r'\1', text)
|
|
|
|
|
|
# Remove emphasis
|
|
|
text = re.sub(r'\*\*(.*?)\*\*', r'\1', text)
|
|
|
text = re.sub(r'\*(.*?)\*', r'\1', text)
|
|
|
text = re.sub(r'__(.*?)__', r'\1', text)
|
|
|
text = re.sub(r'_(.*?)_', r'\1', text)
|
|
|
|
|
|
# Remove code blocks
|
|
|
text = re.sub(r'```(.*?)```', r' \1 ', text, flags=re.DOTALL)
|
|
|
|
|
|
# Remove inline code
|
|
|
text = re.sub(r'`(.*?)`', r' \1 ', text)
|
|
|
|
|
|
# Remove links
|
|
|
text = re.sub(r'\[(.*?)\]\(.*?\)', r'\1', text)
|
|
|
|
|
|
return text.strip()
|
|
|
|
|
|
# 句号列表
|
|
|
stopList = ["。", "！", "？", ".", "!", "?"]
|
|
|
|
|
|
def getWikiSentences(html, line_max_length=150):
|
|
|
# Remove all new-line
|
|
|
html = html.replace("\r", "").replace("\n", "")
|
|
|
|
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
|
parserOutput = soup.find(class_="mw-parser-output")
|
|
|
|
|
|
# Remove <style>
|
|
|
for style in parserOutput.find_all("style"):
|
|
|
style.decompose()
|
|
|
|
|
|
# Remove <caption> if empty
|
|
|
for caption in parserOutput.find_all("caption"):
|
|
|
if caption.string is None or caption.string.strip() == "":
|
|
|
caption.decompose()
|
|
|
|
|
|
mdContent = MarkdownConverter(heading_style="ATX", newline_style="BACKSLASH").convert_soup(parserOutput)
|
|
|
|
|
|
titlePath = []
|
|
|
currentDepth = 0
|
|
|
|
|
|
docList = []
|
|
|
for line in mdContent.split("\n"):
|
|
|
line = line.strip()
|
|
|
if line == "":
|
|
|
continue
|
|
|
if line[0] == "#":
|
|
|
# Title
|
|
|
depth = 0
|
|
|
while line[depth] == "#":
|
|
|
depth += 1
|
|
|
if depth > currentDepth:
|
|
|
for i in range(depth - currentDepth):
|
|
|
titlePath.append(None)
|
|
|
titlePath.append(line[depth:].strip())
|
|
|
else:
|
|
|
titlePath = titlePath[:depth]
|
|
|
titlePath.append(line[depth:].strip())
|
|
|
currentDepth = depth
|
|
|
elif line[0] == "|" and line[-1] == "|":
|
|
|
# Table
|
|
|
continue
|
|
|
else:
|
|
|
if line[0] == ">":
|
|
|
# Quote
|
|
|
line = line[1:].strip()
|
|
|
# Sentence
|
|
|
titlePathStr = "/".join([title for title in titlePath if title is not None])
|
|
|
if titlePathStr == "":
|
|
|
titlePathStr = "简介"
|
|
|
|
|
|
# Split line by stops
|
|
|
sentences = re.split(r"([。！？.!?]+)", line)
|
|
|
|
|
|
combinedSentences = []
|
|
|
lineChunk = []
|
|
|
lineLen = 0
|
|
|
|
|
|
# Combine stops to previous sentence, and combine line untill line_max_length
|
|
|
for i in range(len(sentences) - 1):
|
|
|
if sentences[i] == "":
|
|
|
continue
|
|
|
if re.match(r"^[。！？.!?]+$", sentences[i + 1]):
|
|
|
sentences[i] += sentences[i + 1]
|
|
|
sentences[i + 1] = ""
|
|
|
|
|
|
lineLen += len(sentences[i])
|
|
|
if lineLen > line_max_length:
|
|
|
combinedSentences.append("".join(lineChunk))
|
|
|
lineChunk = []
|
|
|
lineLen = len(sentences[i])
|
|
|
lineChunk.append(sentences[i])
|
|
|
if len(lineChunk) > 0:
|
|
|
combinedSentences.append("".join(lineChunk))
|
|
|
|
|
|
# generate doc list
|
|
|
for text in combinedSentences:
|
|
|
stripedText = stripMarkdown(text)
|
|
|
|
|
|
stripedText = titlePathStr + ": " + stripedText
|
|
|
text = titlePathStr + ": " + text
|
|
|
|
|
|
# Strip markdown syntax
|
|
|
doc = {
|
|
|
"text": stripedText,
|
|
|
"markdown": text,
|
|
|
"text_len": len(stripedText),
|
|
|
"markdown_len": len(text),
|
|
|
}
|
|
|
docList.append(doc)
|
|
|
|
|
|
return docList |