diff --git a/examples/run_language_modeling.py b/examples/run_language_modeling.py
index 5ab9ca31f9..0890598d5e 100644
--- a/examples/run_language_modeling.py
+++ b/examples/run_language_modeling.py
@@ -624,6 +624,7 @@ def main():
         and os.listdir(args.output_dir)
         and args.do_train
         and not args.overwrite_output_dir
+        and not args.should_continue
     ):
         raise ValueError(
             "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(