From 74a24f0fe96d3ca19157171e333cfcc2ab4707c5 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 15 Jul 2019 13:49:01 +0200 Subject: [PATCH] clean up file_utils --- pytorch_transformers/file_utils.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/pytorch_transformers/file_utils.py b/pytorch_transformers/file_utils.py index 1397bd416b..25cdf99837 100644 --- a/pytorch_transformers/file_utils.py +++ b/pytorch_transformers/file_utils.py @@ -258,21 +258,3 @@ def get_from_cache(url, cache_dir=None): logger.info("removing temp file %s", temp_file.name) return cache_path - - -def read_set_from_file(filename): - ''' - Extract a de-duped collection (set) of text from a file. - Expected file format is one item per line. - ''' - collection = set() - with open(filename, 'r', encoding='utf-8') as file_: - for line in file_: - collection.add(line.rstrip()) - return collection - - -def get_file_extension(path, dot=True, lower=True): - ext = os.path.splitext(path)[1] - ext = ext if dot else ext[1:] - return ext.lower() if lower else ext