diff --git a/src/transformers/models/bert/configuration_bert.py b/src/transformers/models/bert/configuration_bert.py
index 861cdfbc8ea6..0d66311915a0 100644
--- a/src/transformers/models/bert/configuration_bert.py
+++ b/src/transformers/models/bert/configuration_bert.py
@@ -172,4 +172,11 @@ def inputs(self) -> Mapping[str, Mapping[int, str]]:
 
     @property
     def outputs(self) -> Mapping[str, Mapping[int, str]]:
-        return OrderedDict([("last_hidden_state", {0: "batch", 1: "sequence"}), ("pooler_output", {0: "batch"})])
+        if self.task == "sequence-classification":
+            return OrderedDict([("logits", {0: "batch"})])
+        elif self.task == "token-classification":
+            return OrderedDict([("logits", {0: "batch", 1: "sequence"})])
+        elif self.task == "question-answering":
+            return OrderedDict([("start_logits", {0: "batch", 1: "sequence"}), ("end_logits", {0: "batch", 1: "sequence"})])
+        else:
+            return OrderedDict([("last_hidden_state", {0: "batch", 1: "sequence"}), ("pooler_output", {0: "batch"})])
diff --git a/src/transformers/models/gpt2/configuration_gpt2.py b/src/transformers/models/gpt2/configuration_gpt2.py
index 41120c94daad..4b48d7c50a06 100644
--- a/src/transformers/models/gpt2/configuration_gpt2.py
+++ b/src/transformers/models/gpt2/configuration_gpt2.py
@@ -191,7 +191,7 @@ def __init__(
 class GPT2OnnxConfig(OnnxConfigWithPast):
     @property
     def inputs(self) -> Mapping[str, Mapping[int, str]]:
-        common_inputs = OrderedDict({"input_ids": {0: "batch"}})
+        common_inputs = OrderedDict({"input_ids": {0: "batch", 1: "sequence"}})
         if self.use_past:
             for i in range(self._config.n_layer * 2):
                 common_inputs[f"past_key_values.{i}"] = {0: "batch", 2: "sequence"}
@@ -204,6 +204,8 @@ def inputs(self) -> Mapping[str, Mapping[int, str]]:
 
     @property
     def outputs(self) -> Mapping[str, Mapping[int, str]]:
+        if self.task == "causal-lm":
+            return OrderedDict([("logits", {0: "batch", 1: "sequence"})])
         common_outputs = OrderedDict({"last_hidden_state": {0: "batch", 1: "sequence"}})
         if self.use_past:
             for i in range(self._config.n_layer * 2):
diff --git a/src/transformers/onnx/features.py b/src/transformers/onnx/features.py
index 92caafb3c5ac..72dc45958996 100644
--- a/src/transformers/onnx/features.py
+++ b/src/transformers/onnx/features.py
@@ -59,11 +59,24 @@ class FeaturesManager:
     # Set of model topologies we support associated to the features supported by each topology and the factory
     _SUPPORTED_MODEL_KIND = {
         "albert": supported_features_mapping("default", onnx_config_cls=AlbertOnnxConfig),
-        "bart": supported_features_mapping("default", onnx_config_cls=BartOnnxConfig),
+        "bart": supported_features_mapping(
+            "default",
+            "sequence-classification",
+            onnx_config_cls=BartOnnxConfig
+        ),
         "mbart": supported_features_mapping("default", onnx_config_cls=MBartOnnxConfig),
-        "bert": supported_features_mapping("default", onnx_config_cls=BertOnnxConfig),
+        "bert": supported_features_mapping(
+            "default",
+            "sequence-classification",
+            "token-classification",
+            "question-answering",
+            onnx_config_cls=BertOnnxConfig
+        ),
         "distilbert": supported_features_mapping("default", onnx_config_cls=DistilBertOnnxConfig),
-        "gpt2": supported_features_mapping("default", onnx_config_cls=GPT2OnnxConfig),
+        "gpt2": supported_features_mapping(
+            "default",
+            "causal-lm",
+            onnx_config_cls=GPT2OnnxConfig),
         "longformer": supported_features_mapping("default", onnx_config_cls=LongformerOnnxConfig),
         "roberta": supported_features_mapping("default", onnx_config_cls=RobertaOnnxConfig),
         "t5": supported_features_mapping(