diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index 827a7ad2a094..e06aaa819929 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -621,7 +621,11 @@ static int dwc3_gadget_start_config(struct dwc3_ep *dep)
 	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 		struct dwc3_ep *dep = dwc->eps[i];
 
-		if (!dep)
+		/*
+		 * Don't set xfer resource with USB GSI endpoint as it is
+		 * performed before enabling USB GSI endpoint.
+		 */
+		if (!dep || dep->gsi)
 			continue;
 
 		ret = dwc3_gadget_set_xfer_resource(dep);