diff --git a/src/transformers/models/deformable_detr/custom_kernel/cpu/ms_deform_attn_cpu.cpp b/src/transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.cpp
similarity index 100%
rename from src/transformers/models/deformable_detr/custom_kernel/cpu/ms_deform_attn_cpu.cpp
rename to src/transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.cpp
diff --git a/src/transformers/models/deformable_detr/custom_kernel/cpu/ms_deform_attn_cpu.h b/src/transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.h
similarity index 100%
rename from src/transformers/models/deformable_detr/custom_kernel/cpu/ms_deform_attn_cpu.h
rename to src/transformers/kernels/deformable_detr/cpu/ms_deform_attn_cpu.h
diff --git a/src/transformers/models/deformable_detr/custom_kernel/cuda/ms_deform_attn_cuda.cu b/src/transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.cu
similarity index 100%
rename from src/transformers/models/deformable_detr/custom_kernel/cuda/ms_deform_attn_cuda.cu
rename to src/transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.cu
diff --git a/src/transformers/models/deformable_detr/custom_kernel/cuda/ms_deform_attn_cuda.cuh b/src/transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.cuh
similarity index 100%
rename from src/transformers/models/deformable_detr/custom_kernel/cuda/ms_deform_attn_cuda.cuh
rename to src/transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.cuh
diff --git a/src/transformers/models/deformable_detr/custom_kernel/cuda/ms_deform_attn_cuda.h b/src/transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.h
similarity index 100%
rename from src/transformers/models/deformable_detr/custom_kernel/cuda/ms_deform_attn_cuda.h
rename to src/transformers/kernels/deformable_detr/cuda/ms_deform_attn_cuda.h
diff --git a/src/transformers/models/deformable_detr/custom_kernel/cuda/ms_deform_im2col_cuda.cuh b/src/transformers/kernels/deformable_detr/cuda/ms_deform_im2col_cuda.cuh
similarity index 100%
rename from src/transformers/models/deformable_detr/custom_kernel/cuda/ms_deform_im2col_cuda.cuh
rename to src/transformers/kernels/deformable_detr/cuda/ms_deform_im2col_cuda.cuh
diff --git a/src/transformers/models/deformable_detr/custom_kernel/ms_deform_attn.h b/src/transformers/kernels/deformable_detr/ms_deform_attn.h
similarity index 100%
rename from src/transformers/models/deformable_detr/custom_kernel/ms_deform_attn.h
rename to src/transformers/kernels/deformable_detr/ms_deform_attn.h
diff --git a/src/transformers/models/deformable_detr/custom_kernel/vision.cpp b/src/transformers/kernels/deformable_detr/vision.cpp
similarity index 100%
rename from src/transformers/models/deformable_detr/custom_kernel/vision.cpp
rename to src/transformers/kernels/deformable_detr/vision.cpp
diff --git a/src/transformers/models/yoso/common.h b/src/transformers/kernels/yoso/common.h
similarity index 100%
rename from src/transformers/models/yoso/common.h
rename to src/transformers/kernels/yoso/common.h
diff --git a/src/transformers/models/yoso/common_cuda.h b/src/transformers/kernels/yoso/common_cuda.h
similarity index 100%
rename from src/transformers/models/yoso/common_cuda.h
rename to src/transformers/kernels/yoso/common_cuda.h
diff --git a/src/transformers/models/yoso/common_cuda_device.h b/src/transformers/kernels/yoso/common_cuda_device.h
similarity index 100%
rename from src/transformers/models/yoso/common_cuda_device.h
rename to src/transformers/kernels/yoso/common_cuda_device.h
diff --git a/src/transformers/models/yoso/fast_lsh_cumulation.cu b/src/transformers/kernels/yoso/fast_lsh_cumulation.cu
similarity index 100%
rename from src/transformers/models/yoso/fast_lsh_cumulation.cu
rename to src/transformers/kernels/yoso/fast_lsh_cumulation.cu
diff --git a/src/transformers/models/yoso/fast_lsh_cumulation.h b/src/transformers/kernels/yoso/fast_lsh_cumulation.h
similarity index 100%
rename from src/transformers/models/yoso/fast_lsh_cumulation.h
rename to src/transformers/kernels/yoso/fast_lsh_cumulation.h
diff --git a/src/transformers/models/yoso/fast_lsh_cumulation_cuda.cu b/src/transformers/kernels/yoso/fast_lsh_cumulation_cuda.cu
similarity index 100%
rename from src/transformers/models/yoso/fast_lsh_cumulation_cuda.cu
rename to src/transformers/kernels/yoso/fast_lsh_cumulation_cuda.cu
diff --git a/src/transformers/models/yoso/fast_lsh_cumulation_cuda.h b/src/transformers/kernels/yoso/fast_lsh_cumulation_cuda.h
similarity index 100%
rename from src/transformers/models/yoso/fast_lsh_cumulation_cuda.h
rename to src/transformers/kernels/yoso/fast_lsh_cumulation_cuda.h
diff --git a/src/transformers/models/yoso/fast_lsh_cumulation_torch.cpp b/src/transformers/kernels/yoso/fast_lsh_cumulation_torch.cpp
similarity index 100%
rename from src/transformers/models/yoso/fast_lsh_cumulation_torch.cpp
rename to src/transformers/kernels/yoso/fast_lsh_cumulation_torch.cpp
diff --git a/src/transformers/models/deformable_detr/load_custom.py b/src/transformers/models/deformable_detr/load_custom.py
index d2a8bc0cb2..c3a822e276 100644
--- a/src/transformers/models/deformable_detr/load_custom.py
+++ b/src/transformers/models/deformable_detr/load_custom.py
@@ -13,16 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ Loading of Deformable DETR's CUDA kernels"""
-
 import os
+from pathlib import Path
 
 
 def load_cuda_kernels():
     from torch.utils.cpp_extension import load
 
-    root = os.path.join(os.path.dirname(os.path.realpath(__file__)), "custom_kernel")
+    root = Path(__file__).resolve().parent.parent.parent / "kernels" / "deformable_detr"
     src_files = [
-        os.path.join(root, filename)
+        root / filename
         for filename in [
             "vision.cpp",
             os.path.join("cpu", "ms_deform_attn_cpu.cpp"),
@@ -33,10 +33,8 @@ def load_cuda_kernels():
     load(
         "MultiScaleDeformableAttention",
         src_files,
-        # verbose=True,
         with_cuda=True,
-        extra_include_paths=[root],
-        # build_directory=os.path.dirname(os.path.realpath(__file__)),
+        extra_include_paths=[str(root)],
         extra_cflags=["-DWITH_CUDA=1"],
         extra_cuda_cflags=[
             "-DCUDA_HAS_FP16=1",
diff --git a/src/transformers/models/yoso/modeling_yoso.py b/src/transformers/models/yoso/modeling_yoso.py
index cf0c814ff9..d549b0080a 100644
--- a/src/transformers/models/yoso/modeling_yoso.py
+++ b/src/transformers/models/yoso/modeling_yoso.py
@@ -16,7 +16,7 @@
 
 
 import math
-import os
+from pathlib import Path
 from typing import Optional, Tuple, Union
 
 import torch
@@ -56,8 +56,8 @@ def load_cuda_kernels():
         from torch.utils.cpp_extension import load
 
         def append_root(files):
-            src_folder = os.path.dirname(os.path.realpath(__file__))
-            return [os.path.join(src_folder, file) for file in files]
+            src_folder = Path(__file__).resolve().parent.parent.parent / "kernels" / "yoso"
+            return [src_folder / file for file in files]
 
         src_files = append_root(
             ["fast_lsh_cumulation_torch.cpp", "fast_lsh_cumulation.cu", "fast_lsh_cumulation_cuda.cu"]
diff --git a/utils/check_build.py b/utils/check_build.py
index a699ed4f7e..e3cca31f83 100644
--- a/utils/check_build.py
+++ b/utils/check_build.py
@@ -21,8 +21,8 @@ from pathlib import Path
 FILES_TO_FIND = [
     "kernels/rwkv/wkv_cuda.cu",
     "kernels/rwkv/wkv_op.cpp",
-    "models/deformable_detr/custom_kernel/ms_deform_attn.h",
-    "models/deformable_detr/custom_kernel/cuda/ms_deform_im2col_cuda.cuh",
+    "kernels/deformable_detr/ms_deform_attn.h",
+    "kernels/deformable_detr/cuda/ms_deform_im2col_cuda.cuh",
     "models/graphormer/algos_graphormer.pyx",
 ]