Validate with official schema, sanitize identifiers properly, panic on duplicate operationId (#39)

* validation schema + sanitize all identifiers

* panic on duplicate operationIds

* renamed sanitized_operation_ids() to sanitize_operation_ids_and_check_duplicate

* added LICENSE and NOTICE for validator_schema

* revert spec

---------

Co-authored-by: Stanislav Kosorin <stanislav.kosorin@cresta.ai>
Co-authored-by: Felix <f.dumbeck@campus.tu-berlin.de>

Author: stano45

Cargo.toml

@@ -17,4 +17,6 @@ gtmpl_value = "0.5.1"
 regex = "1.8.1"
 Inflector = "0.11.4"
 clap = {version = "4.3.0", features = ["derive"]}
+jsonschema = "0.17.0"
+proc-macro2 = "1.0.59"
 

example/specs/basic.yaml

@@ -15,10 +15,10 @@ channels:
         payload:
           type: object
           properties:
-            name:
+            userSingnedUp:
               type: string
     publish:
-      operationId: userSingedUp
+      operationId: userSignedUp
       summary: send welcome email to user
       message:
         payload:

example/specs/invalid-names.yaml

@@ -0,0 +1,50 @@
+asyncapi: 2.1.0
+info:
+  title: My_API
+  version: 1.0.0
+servers:
+  production:
+    url: demo.nats.io
+    protocol: nats
+channels:
+  user/signedup:
+    subscribe:
+      operationId: onUserSignup.l;/.,;';.,\n'
+      summary: User signup notification
+      message:
+        payload:
+          type: object
+          properties:
+            userSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUp:
+              type: string
+    publish:
+      operationId: userSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUp
+      summary: send welcome email to user
+      message:
+        payload:
+          type: string
+
+  user/signedupd:
+    subscribe:
+      operationId: onUserSignup.l/.,;';.,\nfdsfsd
+      summary: User signup notification
+      message:
+        payload:
+          type: object
+          properties:
+            userSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUp:
+              type: string
+    publish:
+      operationId: userSing\edUpuserSing\edUpudserSing\edUpuserSfdsing\edfdsUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUpuserSing\edUp
+      summary: send welcome email to user
+      message:
+        payload:
+          type: string
+  user/buy:
+    subscribe:
+      operationId: userBought
+      summary: User bought something
+      message:
+        payload:
+          type: string
+    

src/main.rs

@@ -17,8 +17,9 @@ fn main() {
     println!("specfile_path: {:?}", specfile_path);
 
     let template_path = Path::new("./templates/");
+    let validator_schema_path = Path::new("./validator_schema/2.1.0.json");
 
-    let spec = parser::parse_spec_to_model(specfile_path).unwrap();
+    let spec = parser::parse_spec_to_model(specfile_path, validator_schema_path).unwrap();
     println!("{:?}", spec);
 
     let title = match args.project_title {

src/parser/common.rs

@@ -1,30 +1,57 @@
-use std::{fs, path::Path};
+use std::{collections::HashSet, fs, path::Path};
 
 use inflector::Inflector;
+use proc_macro2::Ident;
 use regex::Regex;
 
 use crate::asyncapi_model::AsyncAPI;
 
-pub fn parse_spec_to_model(path: &Path) -> Result<AsyncAPI, serde_json::Error> {
-    let string_content = fs::read_to_string(path).expect("file could not be read");
+use super::{
+    preprocessor::{resolve_refs, sanitize_operation_ids_and_check_duplicate},
+    validator::validate_asyncapi_schema,
+};
+
+pub fn parse_spec_to_model(
+    spec_path: &Path,
+    validator_schema_path: &Path,
+) -> Result<AsyncAPI, serde_json::Error> {
+    let spec = parse_string_to_serde_json_value(spec_path);
+    let validator = parse_string_to_serde_json_value(validator_schema_path);
+
+    validate_asyncapi_schema(&validator, &spec);
+
+    let preprocessed_spec = preprocess_schema(spec);
+    let spec = serde_json::from_value::<AsyncAPI>(preprocessed_spec)?;
+    Ok(spec)
+}
+
+fn preprocess_schema(spec: serde_json::Value) -> serde_json::Value {
+    let resolved_refs = resolve_refs(spec.clone(), spec);
+    let mut seen = HashSet::new();
+    let sanitized =
+        sanitize_operation_ids_and_check_duplicate(resolved_refs.clone(), resolved_refs, &mut seen);
+    println!("Preprocessed spec: {}", sanitized);
+    sanitized
+}
+
+fn parse_string_to_serde_json_value(file_path: &Path) -> serde_json::Value {
+    let file_string = fs::read_to_string(file_path).expect("File could not be read");
     // check if file is yaml or json
-    let parsed = match path.extension() {
+    let parsed_value = match file_path.extension() {
         Some(ext) => match ext.to_str() {
-            Some("yaml") => serde_yaml::from_str::<serde_json::Value>(&string_content).unwrap(),
-            Some("yml") => serde_yaml::from_str::<serde_json::Value>(&string_content).unwrap(),
-            Some("json") => serde_json::from_str::<serde_json::Value>(&string_content).unwrap(),
+            Some("yaml") | Some("yml") => {
+                serde_yaml::from_str::<serde_json::Value>(&file_string).unwrap()
+            }
+            Some("json") => serde_json::from_str::<serde_json::Value>(&file_string).unwrap(),
             _ => {
-                panic!("file has no extension");
+                panic!("File has an unsupported extension");
             }
         },
         None => {
-            panic!("file has no extension");
+            panic!("File has no extension");
         }
     };
-    let with_resolved_references =
-        crate::parser::resolve_refs::resolve_refs(parsed.clone(), parsed);
-    let spec = serde_json::from_value::<AsyncAPI>(with_resolved_references)?;
-    Ok(spec)
+    parsed_value
 }
 
 fn capitalize_first_char(s: &str) -> String {
@@ -35,11 +62,16 @@ fn capitalize_first_char(s: &str) -> String {
     }
 }
 
-pub fn convert_string_to_valid_type_name(s: &str, suffix: &str) -> String {
-    let re = Regex::new(r"[^\w\s]").unwrap();
+pub fn validate_identifier_string(s: &str) -> String {
     // Remove special chars, capitalize words, remove spaces
-    let mut root_msg_name = re.replace_all(s, " ").to_title_case().replace(' ', "");
-    // Append Message to the end of the name
-    root_msg_name.push_str(suffix);
-    capitalize_first_char(root_msg_name.as_str())
+    let re = Regex::new(r"[^\w\s]").unwrap();
+    let sanitized_identifier = re.replace_all(s, " ").to_title_case().replace(' ', "");
+    let capitalized_sanitized_identifier = capitalize_first_char(sanitized_identifier.as_str());
+    // Create a new identifier
+    // This acts as validation for the message name, panics when the name is invalid
+    Ident::new(
+        &capitalized_sanitized_identifier,
+        proc_macro2::Span::call_site(),
+    );
+    capitalized_sanitized_identifier
 }

src/parser/mod.rs

@@ -1,8 +1,7 @@
 mod common;
+mod preprocessor;
 mod pubsub;
-mod resolve_refs;
 mod schema_parser;
+mod validator;
 pub use common::parse_spec_to_model;
 pub use pubsub::spec_to_pubsub_template_type;
-pub use resolve_refs::resolve_refs;
-pub use schema_parser::schema_parser_mapper;

src/parser/resolve_refs.rs src/parser/preprocessor.rssrc/parser/resolve_refs.rs renamed to src/parser/preprocessor.rs

@@ -1,3 +1,7 @@
+use crate::parser::common::validate_identifier_string;
+use serde_json::json;
+use std::collections::HashSet;
+
 pub fn resolve_json_path(json: serde_json::Value, path: &str) -> serde_json::Value {
     let parts = path.split('/').collect::<Vec<&str>>();
     let mut current_json = json;
@@ -7,6 +11,57 @@ pub fn resolve_json_path(json: serde_json::Value, path: &str) -> serde_json::Val
     current_json
 }
 
+pub fn sanitize_operation_ids_and_check_duplicate(
+    json: serde_json::Value,
+    root_json: serde_json::Value,
+    seen_operation_ids: &mut HashSet<String>,
+) -> serde_json::Value {
+    match json {
+        serde_json::Value::Object(map) => {
+            let mut new_map = serde_json::Map::new();
+            for (key, value) in map {
+                if key == "operationId" {
+                    if let serde_json::Value::String(string_val) = &value {
+                        let sanitized_val = validate_identifier_string(string_val.as_str());
+                        if seen_operation_ids.contains(&sanitized_val) {
+                            panic!("Duplicate operationId found: {}", sanitized_val);
+                        } else {
+                            seen_operation_ids.insert(sanitized_val.clone());
+                            new_map.insert(key, json!(sanitized_val));
+                        }
+                    } else {
+                        panic!("operationId value is not a string");
+                    }
+                } else {
+                    new_map.insert(
+                        key,
+                        sanitize_operation_ids_and_check_duplicate(
+                            value,
+                            root_json.clone(),
+                            seen_operation_ids,
+                        ),
+                    );
+                }
+            }
+            serde_json::Value::Object(new_map)
+        }
+        serde_json::Value::Array(array) => {
+            let new_array = array
+                .into_iter()
+                .map(|value| {
+                    sanitize_operation_ids_and_check_duplicate(
+                        value,
+                        root_json.clone(),
+                        seen_operation_ids,
+                    )
+                })
+                .collect();
+            serde_json::Value::Array(new_array)
+        }
+        _ => json,
+    }
+}
+
 pub fn resolve_refs(json: serde_json::Value, root_json: serde_json::Value) -> serde_json::Value {
     match json {
         serde_json::Value::Object(map) => {

src/parser/pubsub.rs

@@ -1,7 +1,7 @@
-use super::{schema_parser::SchemaParserError, schema_parser_mapper};
+use super::schema_parser::{schema_parser_mapper, SchemaParserError};
 use crate::{
     asyncapi_model::{AsyncAPI, OperationMessageType, Payload, ReferenceOr, Schema},
-    parser::common::convert_string_to_valid_type_name,
+    parser::common::validate_identifier_string,
     template_model::PubsubTemplate,
 };
 use std::{collections::HashMap, io};
@@ -37,7 +37,6 @@ fn parse_single_message_operation_type(
     match message_ref_or_item {
         ReferenceOr::Item(message) => match &message.payload {
             Some(Payload::Schema(schema)) => {
-                println!("\nmap schema: {:?}", schema);
                 transform_schema_to_string_vec(schema, &root_msg_name).unwrap()
             }
             Some(Payload::Any(val)) => {
@@ -64,7 +63,7 @@ fn extract_schemas_from_asyncapi(spec: &AsyncAPI) -> Vec<String> {
     return channels_ops
         .iter()
         .flat_map(|x| {
-            let root_msg_name = convert_string_to_valid_type_name(x.0, "");
+            let root_msg_name = validate_identifier_string(x.0);
             let channel = x.1;
             let operation_message = channel.message.as_ref().unwrap();
             match operation_message {

src/parser/schema_parser.rs

@@ -5,7 +5,7 @@ use crate::asyncapi_model::{
 use core::fmt;
 use std::{collections::HashMap, format};
 
-use super::common::convert_string_to_valid_type_name;
+use super::common::validate_identifier_string;
 
 #[derive(Debug, Clone)]
 pub enum SchemaParserError {
@@ -38,7 +38,7 @@ fn object_schema_to_string(
 ) -> Result<String, SchemaParserError> {
     let before_string = format!(
         "#[derive(Clone, Debug, Deserialize, Serialize)]\npub struct {} {{\n",
-        convert_string_to_valid_type_name(property_name, "")
+        validate_identifier_string(property_name)
     );
     let after_string = String::from("\n}\n");
     let property_string_iterator: Vec<Result<String, SchemaParserError>> = schema
@@ -68,21 +68,16 @@ fn object_schema_to_string(
     Ok(property_name.to_string())
 }
 
-fn sanitize_property_name(property_name: &str) -> String {
-    // TODO: do proper sanitization so that the property name is a valid rust identifier
-    property_name.replace('-', "_")
-}
-
 fn primitive_type_to_string(
     schema_type: Type,
     property_name: &str,
 ) -> Result<String, SchemaParserError> {
     // TODO: Add support for arrays
     match schema_type {
-        Type::String(_var) => Ok(format!("pub {}: String", sanitize_property_name(property_name))),
-        Type::Number(_var) => Ok(format!("pub {}: f64", sanitize_property_name(property_name))),
-        Type::Integer(_var) => Ok(format!("pub {}: int64", sanitize_property_name(property_name)) ),
-        Type::Boolean{} => Ok(format!("pub {}: bool", sanitize_property_name(property_name))),
+        Type::String(_var) => Ok(format!("pub {}: String", validate_identifier_string(property_name))),
+        Type::Number(_var) => Ok(format!("pub {}: f64", validate_identifier_string(property_name))),
+        Type::Integer(_var) => Ok(format!("pub {}: int64", validate_identifier_string(property_name)) ),
+        Type::Boolean{} => Ok(format!("pub {}: bool", validate_identifier_string(property_name))),
         _type => Err(SchemaParserError::GenericError("Unsupported primitive type: Currently only supports string, number, integer and boolean types".to_string(), Some(property_name.into()))),
     }
 }
@@ -99,8 +94,8 @@ pub fn schema_parser_mapper(
                 let struct_name = object_schema_to_string(y, property_name, all_structs)?;
                 Ok(format!(
                     "pub {}: {}",
-                    property_name,
-                    convert_string_to_valid_type_name(struct_name.as_str(), "").as_str()
+                    struct_name,
+                    validate_identifier_string(struct_name.as_str()).as_str()
                 ))
             }
             _primitive_type => primitive_type_to_string(_primitive_type.clone(), property_name),

src/parser/validator.rs

@@ -0,0 +1,15 @@
+use jsonschema::JSONSchema;
+
+pub fn validate_asyncapi_schema(validator: &serde_json::Value, instance: &serde_json::Value) {
+    let compiled = JSONSchema::compile(validator).expect("A valid schema");
+    let result = compiled.validate(instance);
+    if let Err(errors) = result {
+        for error in errors {
+            println!("Validation error: {}", error);
+            println!("Instance path: {}", error.instance_path);
+        }
+        panic!("Validation failed");
+    } else {
+        println!("Validation succeeded");
+    }
+}