Handle the case when title is None (#7941)
This commit is contained in:
@@ -35,9 +35,10 @@ def split_documents(documents: dict) -> dict:
|
|||||||
"""Split documents into passages"""
|
"""Split documents into passages"""
|
||||||
titles, texts = [], []
|
titles, texts = [], []
|
||||||
for title, text in zip(documents["title"], documents["text"]):
|
for title, text in zip(documents["title"], documents["text"]):
|
||||||
for passage in split_text(text):
|
if text is not None:
|
||||||
titles.append(title)
|
for passage in split_text(text):
|
||||||
texts.append(passage)
|
titles.append(title if title is not None else "")
|
||||||
|
texts.append(passage)
|
||||||
return {"title": titles, "text": texts}
|
return {"title": titles, "text": texts}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user