Remove-auth-token (#27060)

* don't use `use_auth_token`internally

* let's use token everywhere

* fixup
This commit is contained in:
Arthur
2023-11-13 14:20:54 +01:00
committed by GitHub
parent 8f577dca4f
commit b97cab7e6d
29 changed files with 93 additions and 101 deletions

View File

@@ -65,7 +65,7 @@ def normalize_text(text: str) -> str:
def main(args):
# load dataset
dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True)
dataset = load_dataset(args.dataset, args.config, split=args.split, token=True)
# for testing: only process the first two examples as a test
# dataset = dataset.select(range(10))