Experimental new docs output

Makefile

@@ -52,14 +52,14 @@ transform-expand-generics: ## Create a new schema with all generics expanded
 	@npm run transform-expand-generics --prefix compiler
 
 transform-to-openapi: ## Generate the OpenAPI definition from the compiled schema
-	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --output output/openapi/elasticsearch-openapi.json
-	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --output output/openapi/elasticsearch-openapi.json --branch main
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --output output/openapi/elasticsearch-serverless-openapi.json --branch main
 
 transform-to-openapi-for-docs: ## Generate the OpenAPI definition tailored for API docs generation
 	@make generate-language-examples
 	@make generate
-	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --include-language-examples --output output/openapi/elasticsearch-openapi-docs.json
-	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --lift-enum-descriptions --merge-multipath-endpoints --multipath-redirects --include-language-examples --output output/openapi/elasticsearch-serverless-openapi-docs.json
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor stack --lift-enum-descriptions --include-language-examples --branch $(branch) --output output/openapi/elasticsearch-openapi-docs.json
+	@npm run transform-to-openapi -- --schema output/schema/schema.json --flavor serverless --lift-enum-descriptions --include-language-examples --branch $(branch) --output output/openapi/elasticsearch-serverless-openapi-docs.json
 
 filter-for-serverless: ## Generate the serverless version from the compiled schema
 	@npm run --prefix compiler filter-by-availability -- --serverless --visibility=public --input ../output/schema/schema.json --output ../output/output/openapi/elasticsearch-serverless-openapi.json
@@ -78,7 +78,7 @@ overlay-docs: ## Apply overlays to OpenAPI documents
 	rm output/openapi/elasticsearch-openapi-docs.tmp*.json
 
 generate-language-examples:
-	@npm update --prefix docs/examples @elastic/request-converter
+	@npm update --prefix docs/examples @elastic/request-converter || echo "Warning: could not update @elastic/request-converter, using existing version"
 	@node docs/examples/generate-language-examples.js
 	@npm run format:fix-examples --prefix compiler
 

compiler-rs/clients_schema/src/lib.rs

@@ -953,6 +953,9 @@ pub struct Endpoint {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub doc_tag: Option<String>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub category: Option<String>,
+
     /// If missing, there is not yet a request definition for this endpoint.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub request: Option<TypeName>,
@@ -1028,11 +1031,24 @@ pub struct UrlTemplate {
     pub deprecation: Option<Deprecation>,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct FlavorInfo {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub title: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct ModelInfo {
     pub title: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
     pub license: License,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub flavors: Option<IndexMap<Flavor, FlavorInfo>>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -1042,11 +1058,66 @@ pub struct License {
     pub url: String,
 }
 
+/// One OpenAPI 3 [Security Requirement Object](https://swagger.io/specification/#security-requirement-object):
+/// scheme name to optional OAuth2 scopes (empty vec when scopes do not apply).
+pub type OpenApiSecurityRequirement = IndexMap<String, Vec<String>>;
+
+/// Document-level `security` array: each element is an alternative set of requirements (OR); schemes
+/// within one [`OpenApiSecurityRequirement`] are ANDed.
+pub type OpenApiSecurityList = Vec<OpenApiSecurityRequirement>;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct OpenApiFlavorSecurityBlock {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub security: Option<OpenApiSecurityList>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub security_schemes: Option<serde_json::Value>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ExternalDocs {
+    pub url: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TagMetadata {
+    pub display_name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub external_docs: Option<ExternalDocs>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TagGroup {
+    pub name: String,
+    pub tags: Vec<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct OpenApiMetadata {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub flavors: Option<IndexMap<String, OpenApiFlavorSecurityBlock>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tag_metadata: Option<IndexMap<String, TagMetadata>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub tag_groups: Option<Vec<TagGroup>>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct Model {
     #[serde(rename = "_info", skip_serializing_if = "Option::is_none")]
     pub info: Option<ModelInfo>,
+    #[serde(rename = "_openapi", skip_serializing_if = "Option::is_none")]
+    pub openapi: Option<OpenApiMetadata>,
     pub endpoints: Vec<Endpoint>,
     pub types: Vec<TypeDefinition>,
 }
@@ -1067,6 +1138,9 @@ pub struct IndexedModel {
     #[serde(rename = "_info", skip_serializing_if = "Option::is_none")]
     pub info: Option<ModelInfo>,
 
+    #[serde(rename = "_openapi", skip_serializing_if = "Option::is_none")]
+    pub openapi: Option<OpenApiMetadata>,
+
     pub endpoints: Vec<Endpoint>,
 
     #[serde(serialize_with = "serialize_types")]
@@ -1183,6 +1257,7 @@ impl From<Model> for IndexedModel {
     fn from(value: Model) -> Self {
         IndexedModel {
             info: value.info,
+            openapi: value.openapi,
             endpoints: value.endpoints,
             types: value.types.into_iter().map(|t| (t.name().clone(), t)).collect(),
         }
@@ -1193,6 +1268,7 @@ impl From<IndexedModel> for Model {
     fn from(value: IndexedModel) -> Model {
         Model {
             info: value.info,
+            openapi: value.openapi,
             endpoints: value.endpoints,
             types: value.types.into_iter().map(|(_, t)| t).collect(),
         }

compiler-rs/clients_schema/src/transform/expand_generics.rs

@@ -498,14 +498,28 @@ mod tests {
 
     #[test]
     pub fn compare_with_js_version() -> testresult::TestResult {
+        let manifest_dir = env!("CARGO_MANIFEST_DIR");
+        let workspace_root = std::path::Path::new(manifest_dir)
+            .parent()
+            .and_then(|p| p.parent())
+            .ok_or("Failed to determine workspace root")?;
+
+        let canonical_file = workspace_root.join("output/schema/schema-no-generics.json");
+
+        // Skip test if canonical file doesn't exist (happens in CI when generics expansion hasn't been run separately)
+        if !canonical_file.exists() {
+            return Ok(());
+        }
+
         let canonical_json = {
             // Deserialize and reserialize to have a consistent JSON format
-            let json = std::fs::read_to_string("../../output/schema/schema-no-generics.json")?;
+            let json = std::fs::read_to_string(&canonical_file)?;
             let model: IndexedModel = serde_json::from_str(&json)?;
             serde_json::to_string_pretty(&model)?
         };
 
-        let schema_json = std::fs::read_to_string("../../output/schema/schema.json")?;
+        let schema_path = workspace_root.join("output/schema/schema.json");
+        let schema_json = std::fs::read_to_string(schema_path)?;
         let model: IndexedModel = serde_json::from_str(&schema_json)?;
         let model = expand(model, ExpandConfig::default())?;
 

compiler-rs/clients_schema_to_openapi/src/cli.rs

@@ -32,31 +32,11 @@ pub struct Cli {
     #[argh(switch)]
     pub lift_enum_descriptions: bool,
 
-    /// merge endpoints with multiple paths into a single OpenAPI operation [default = false]
-    #[argh(switch)]
-    pub merge_multipath_endpoints: bool,
-
-    /// output a redirection map when merging multipath endpoints
-    #[argh(switch)]
-    pub multipath_redirects: bool,
-
     /// include the x-codeSamples extension with language examples for all endpoints
     #[argh(switch)]
     pub include_language_examples: bool,
 }
 
-impl Cli {
-    pub fn redirect_path(&self, output: &PathBuf) -> Option<String> {
-        if self.multipath_redirects {
-            let path = output.to_string_lossy();
-            let path = path.rsplit_once('.').unwrap().0;
-            Some(format!("{}.redirects.csv", path))
-        } else {
-            None
-        }
-    }
-}
-
 use derive_more::FromStr;
 
 #[derive(Debug, Clone, PartialEq, FromStr)]
@@ -83,8 +63,6 @@ impl From<Cli> for Configuration {
             flavor,
             branch,
             lift_enum_descriptions: cli.lift_enum_descriptions,
-            merge_multipath_endpoints: cli.merge_multipath_endpoints,
-            multipath_redirects: cli.multipath_redirects,
             include_language_examples: cli.include_language_examples,
             namespaces: if cli.namespace.is_empty() {
                 None

compiler-rs/clients_schema_to_openapi/src/components.rs

@@ -15,11 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::BTreeMap;
+use crate::utils::SchemaName;
+use crate::Configuration;
 use clients_schema::TypeName;
 use openapiv3::{Components, Parameter, ReferenceOr, RequestBody, Response, Schema, StatusCode};
-use crate::Configuration;
-use crate::utils::SchemaName;
 
 // Separator used to combine parts of a component path.
 // OpenAPI says `$ref` must comply with RFC 3968 (escaping reserved chars),
@@ -33,19 +32,11 @@ pub struct TypesAndComponents<'a> {
     pub config: &'a Configuration,
     pub model: &'a clients_schema::IndexedModel,
     pub components: &'a mut Components,
-    // Redirections (if paths multipaths endpoints are merged)
-    pub redirects: Option<BTreeMap<String, String>>,
 }
 
 impl<'a> TypesAndComponents<'a> {
     pub fn new(config: &'a Configuration, model: &'a clients_schema::IndexedModel, components: &'a mut Components) -> TypesAndComponents<'a> {
-        let redirects = if config.merge_multipath_endpoints && config.multipath_redirects {
-            Some(BTreeMap::new())
-        } else {
-            None
-        };
-
-        TypesAndComponents { config, model, components, redirects }
+        TypesAndComponents { config, model, components }
     }
 
     pub fn add_request_body(&mut self, endpoint: &str, body: RequestBody) -> ReferenceOr<RequestBody> {