Handle the case when title is None (#7941)

This commit is contained in:
Lalit Pagaria
2020-10-23 15:54:45 +02:00
committed by GitHub
parent 023f0f3708
commit 88b3a91e61

View File

@@ -35,9 +35,10 @@ def split_documents(documents: dict) -> dict:
"""Split documents into passages"""
titles, texts = [], []
for title, text in zip(documents["title"], documents["text"]):
for passage in split_text(text):
titles.append(title)
texts.append(passage)
if text is not None:
for passage in split_text(text):
titles.append(title if title is not None else "")
texts.append(passage)
return {"title": titles, "text": texts}