Generate: move generation_*.py src files into generation/*.py (#20096)

* move generation_*.py src files into generation/*.py

* populate generation.__init__ with lazy loading

* move imports and references from generation.xxx.object to generation.object

tests/generation/test_beam_constraints.py

@@ -23,7 +23,7 @@ from transformers.testing_utils import require_torch
 if is_torch_available():
     import torch
 
-    from transformers.generation_beam_constraints import DisjunctiveConstraint
+    from transformers.generation import DisjunctiveConstraint
 
 
 @require_torch

tests/generation/test_beam_search.py

@@ -25,8 +25,13 @@ from ..test_modeling_common import floats_tensor, ids_tensor
 if is_torch_available():
     import torch
 
-    from transformers.generation_beam_constraints import DisjunctiveConstraint, PhrasalConstraint
-    from transformers.generation_beam_search import BeamHypotheses, BeamSearchScorer, ConstrainedBeamSearchScorer
+    from transformers.generation import (
+        BeamHypotheses,
+        BeamSearchScorer,
+        ConstrainedBeamSearchScorer,
+        DisjunctiveConstraint,
+        PhrasalConstraint,
+    )
 
 
 class BeamSearchTester:

tests/generation/test_flax_logits_process.py

@@ -27,7 +27,7 @@ from ..test_modeling_flax_common import ids_tensor
 if is_flax_available():
     import jax
     import jax.numpy as jnp
-    from transformers.generation_flax_logits_process import (
+    from transformers.generation import (
         FlaxForcedBOSTokenLogitsProcessor,
         FlaxForcedEOSTokenLogitsProcessor,
         FlaxLogitsProcessorList,

tests/generation/test_logits_process.py

@@ -26,7 +26,7 @@ if is_torch_available():
     import torch
     from torch import nn
 
-    from transformers.generation_logits_process import (
+    from transformers.generation import (
         EncoderNoRepeatNGramLogitsProcessor,
         ExponentialDecayLengthPenalty,
         ForcedBOSTokenLogitsProcessor,

tests/generation/test_stopping_criteria.py

@@ -25,7 +25,7 @@ from ..test_modeling_common import ids_tensor
 if is_torch_available():
     import torch
 
-    from transformers.generation_stopping_criteria import (
+    from transformers.generation import (
         MaxLengthCriteria,
         MaxNewTokensCriteria,
         MaxTimeCriteria,

tests/generation/test_tf_logits_process.py

@@ -26,7 +26,7 @@ from transformers.testing_utils import require_tf
 if is_tf_available():
     import tensorflow as tf
 
-    from transformers.generation_tf_logits_process import (
+    from transformers.generation import (
         TFForcedBOSTokenLogitsProcessor,
         TFForcedEOSTokenLogitsProcessor,
         TFForceTokensLogitsProcessor,

tests/generation/test_utils.py

@@ -42,32 +42,34 @@ if is_torch_available():
         pipeline,
         top_k_top_p_filtering,
     )
-    from transformers.generation_beam_constraints import DisjunctiveConstraint, PhrasalConstraint
-    from transformers.generation_beam_search import BeamSearchScorer, ConstrainedBeamSearchScorer
-    from transformers.generation_logits_process import (
-        ForcedBOSTokenLogitsProcessor,
-        ForcedEOSTokenLogitsProcessor,
-        HammingDiversityLogitsProcessor,
-        InfNanRemoveLogitsProcessor,
-        LogitsProcessorList,
-        MinLengthLogitsProcessor,
-        NoBadWordsLogitsProcessor,
-        NoRepeatNGramLogitsProcessor,
-        RepetitionPenaltyLogitsProcessor,
-        TemperatureLogitsWarper,
-        TopKLogitsWarper,
-        TopPLogitsWarper,
-    )
-    from transformers.generation_stopping_criteria import MaxLengthCriteria, StoppingCriteria, StoppingCriteriaList
-    from transformers.generation_utils import (
+    from transformers.generation import (
         BeamSampleDecoderOnlyOutput,
         BeamSampleEncoderDecoderOutput,
         BeamSearchDecoderOnlyOutput,
         BeamSearchEncoderDecoderOutput,
+        BeamSearchScorer,
+        ConstrainedBeamSearchScorer,
+        DisjunctiveConstraint,
+        ForcedBOSTokenLogitsProcessor,
+        ForcedEOSTokenLogitsProcessor,
         GreedySearchDecoderOnlyOutput,
         GreedySearchEncoderDecoderOutput,
+        HammingDiversityLogitsProcessor,
+        InfNanRemoveLogitsProcessor,
+        LogitsProcessorList,
+        MaxLengthCriteria,
+        MinLengthLogitsProcessor,
+        NoBadWordsLogitsProcessor,
+        NoRepeatNGramLogitsProcessor,
+        PhrasalConstraint,
+        RepetitionPenaltyLogitsProcessor,
         SampleDecoderOnlyOutput,
         SampleEncoderDecoderOutput,
+        StoppingCriteria,
+        StoppingCriteriaList,
+        TemperatureLogitsWarper,
+        TopKLogitsWarper,
+        TopPLogitsWarper,
     )