Skip to content

Commit 4bd698c

Browse files
esponaespona
committed
Implemented transformer for asset to cbm dataset (WIP).
1 parent 5676509 commit 4bd698c

4 files changed

Lines changed: 237 additions & 51 deletions

File tree

‎providers/provider/src/main/java/org/eclipse/edc/heleade/provider/extension/content/based/api/asset/AssetJsonObjectToCbmJsonObjectTransformer.java‎

Lines changed: 22 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818
import jakarta.json.JsonArrayBuilder;
1919
import jakarta.json.JsonObject;
2020
import jakarta.json.JsonObjectBuilder;
21-
import jakarta.json.JsonString;
22-
import jakarta.json.JsonValue;
2321
import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset;
2422
import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset;
2523
import org.eclipse.edc.heleade.commons.content.based.catalog.CbmConstants;
@@ -30,6 +28,7 @@
3028

3129
import static org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset.EDC_ASSET_TYPE;
3230
import static org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset.PROPERTY_ID;
31+
import static org.eclipse.edc.heleade.commons.content.based.catalog.CbmConstants.CBM_SCHEMA;
3332
import static org.eclipse.edc.jsonld.spi.Namespaces.DCAT_SCHEMA;
3433
import static org.eclipse.edc.jsonld.spi.Namespaces.DCT_SCHEMA;
3534
import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATASET_TYPE;
@@ -93,73 +92,60 @@ public static JsonObject transformAssetToCbmJsonObject(String id, JsonObject jso
9392
.add("@id", id)
9493
.add("@type", DCAT_DATASET_TYPE);
9594

95+
// Create already the distribution builder
96+
JsonArrayBuilder distributionArrayBuilder = Json.createArrayBuilder();
97+
JsonObjectBuilder distributionBuilder = Json.createObjectBuilder();
98+
9699
// Add context from original object
97100
if (jsonObject.containsKey("@context")) {
98101
datasetBuilder.add("@context", jsonObject.get("@context"));
99102
}
100103

101-
var propertiesValue = jsonObject.get(EDC_NAMESPACE + "properties");
102-
JsonObject properties = getObjectOrFirstInArray(jsonObject.get(EDC_NAMESPACE + "properties"));
104+
JsonObject properties = jsonObject.getJsonObject(EDC_NAMESPACE + "properties");
103105

104106
properties.forEach((key, value) -> {
105107
if (!isForbiddenProperty(key)) {
106108
datasetBuilder.add(key, value);
109+
} else if ((DCAT_SCHEMA + "byteSize").equals(key) || (CBM_SCHEMA + "hasDataDictionary").equals(key)) {
110+
distributionBuilder.add(key, value);
107111
}
108112
});
109113

110114
// build the data distribution
111-
JsonArrayBuilder distributionArrayBuilder = Json.createArrayBuilder();
112-
JsonObject dataAddress = getObjectOrFirstInArray(jsonObject.get(EDC_NAMESPACE + "dataAddress"));
113-
114-
JsonObjectBuilder distributionBuilder = Json.createObjectBuilder();
115+
JsonObject dataAddress = jsonObject.getJsonObject(EDC_NAMESPACE + "dataAddress");
115116
JsonObjectBuilder format = Json.createObjectBuilder();
116-
117117
distributionBuilder.add("@type", DCAT_SCHEMA + "Distribution");
118118

119119
// get format and type
120-
String dataAddressType = getStringOrFirstInArray(dataAddress.get(EDC_NAMESPACE + "type"));
120+
String dataAddressType = dataAddress.getString(EDC_NAMESPACE + "type");
121121
format.add("@id", dataAddressType);
122-
format.add(DCT_SCHEMA + "type", format.build());
123122
distributionBuilder.add(DCT_SCHEMA + "format", format.build());
124123

125124
// get url
126-
var baseUrlValue = dataAddress.get(EDC_NAMESPACE + "baseUrl");
127-
if (baseUrlValue != null) {
128-
String baseUrl = getStringOrFirstInArray(baseUrlValue);
125+
var baseUrl = dataAddress.getString(EDC_NAMESPACE + "baseUrl");
126+
if (baseUrl != null && !baseUrl.isEmpty()) {
129127
distributionBuilder.add(DCAT_SCHEMA + "accessURL", baseUrl);
130128
}
131129

132-
distributionArrayBuilder.add(distributionBuilder.build());
133-
130+
dataAddress.forEach((key, value) -> {
131+
if (!isForbiddenDistributionProperty(key)) {
132+
distributionBuilder.add(key, value);
133+
}
134+
});
134135

136+
distributionArrayBuilder.add(distributionBuilder.build());
135137
datasetBuilder.add(CbmConstants.DISTRIBUTION_TAG, distributionArrayBuilder.build());
136138

137139
return datasetBuilder.build();
138140
}
139141

140142
private static boolean isForbiddenProperty(String key) {
141143
// Properties that are forbidden
142-
return "id".equals(key) || PROPERTY_ID.equals(key) || key.startsWith("@");
144+
return "id".equals(key) || PROPERTY_ID.equals(key) || key.startsWith("@") || (DCAT_SCHEMA + "byteSize").equals(key) || (CBM_SCHEMA + "hasDataDictionary").equals(key);
143145
}
144146

145-
private static JsonObject getObjectOrFirstInArray(JsonValue value) {
146-
if (value.getValueType() == JsonValue.ValueType.ARRAY) {
147-
return value.asJsonArray().getJsonObject(0);
148-
} else {
149-
return value.asJsonObject();
150-
}
151-
}
152-
153-
private static String getStringOrFirstInArray(JsonValue value) {
154-
if (value.getValueType() == JsonValue.ValueType.ARRAY) {
155-
JsonObject jsonObject = value.asJsonArray().getJsonObject(0);
156-
return jsonObject.getString("@value");
157-
} else if (value.getValueType() == JsonValue.ValueType.OBJECT) {
158-
return value.asJsonObject().getString("@value");
159-
} else if (value.getValueType() == JsonValue.ValueType.STRING) {
160-
return ((JsonString) value).getString();
161-
}
162-
return null;
147+
private static boolean isForbiddenDistributionProperty(String key) {
148+
// Properties that are forbidden
149+
return (EDC_NAMESPACE + "type").equals(key) || (EDC_NAMESPACE + "baseUrl").equals(key) || key.startsWith("@");
163150
}
164-
165151
}

‎providers/provider/src/test/java/org/eclipse/edc/heleade/provider/extension/validation/AssetJsonTransformerTest.java‎

Lines changed: 60 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919
import jakarta.json.JsonObject;
2020
import org.eclipse.edc.heleade.provider.extension.content.based.api.asset.AssetJsonObjectToCbmJsonObjectTransformer;
2121
import org.eclipse.edc.heleade.provider.extension.content.based.api.asset.CbmJsonObjectToAssetJsonObjectTransformer;
22+
import org.eclipse.edc.jsonld.JsonLdExtension;
2223
import org.eclipse.edc.jsonld.spi.JsonLd;
24+
import org.eclipse.edc.junit.extensions.TestServiceExtensionContext;
2325
import org.eclipse.edc.junit.testfixtures.TestUtils;
2426
import org.eclipse.edc.validator.spi.Validator;
2527
import org.junit.jupiter.api.Assertions;
@@ -33,43 +35,64 @@
3335
import java.nio.file.Paths;
3436

3537
import static org.eclipse.edc.heleade.provider.extension.validation.AssetJsonSchemaValidatorTest.getJsonSchema;
36-
import static org.eclipse.edc.heleade.provider.extension.validation.AssetJsonSchemaValidatorTest.initializeJsonLd;
38+
import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.VOCAB;
3739
import static org.eclipse.edc.junit.assertions.AbstractResultAssert.assertThat;
40+
import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE;
3841

3942
public class AssetJsonTransformerTest {
40-
private static final JsonLd JSON_LD = initializeJsonLd();
43+
private static final JsonLd JSON_LD_EMPTY = new JsonLdExtension().createJsonLdService(TestServiceExtensionContext.testServiceExtensionContext());
44+
private static final JsonLd JSON_LD_EDC = new JsonLdExtension().createJsonLdService(TestServiceExtensionContext.testServiceExtensionContext());
4145
private static final String ASSET_SCHEMA_PATH = "providers/provider/src/main/resources/asset-schema-cbm.json";
4246
private static final String DATASET_SCHEMA_PATH = "providers/provider/src/main/resources/dataset-schema-cbm.json";
4347
private static final String RESOURCES_PATH = "providers/provider/";
4448
private static final String MINIMAL_ASSET_PATH = RESOURCES_PATH + "src/test/resources/minimal-edc-asset.json";
4549
private static final String MINIMAL_CBM_PATH = RESOURCES_PATH + "src/test/resources/minimal-cbm-dataset.json";
4650
private static final String MINIMAL_ID = "datasetMandatory";
47-
private static String minimalAssetString;
48-
private static String minimalCbmString;
51+
private static final String FULL_ASSET_PATH = RESOURCES_PATH + "src/test/resources/full-edc-asset.json";
52+
private static final String FULL_CBM_PATH = RESOURCES_PATH + "src/test/resources/full-cbm-dataset.json";
53+
private static final String FULL_ID = "datasetFull";
4954
private static Validator<JsonObject> assetValidator;
5055
private static Validator<JsonObject> datasetValidator;
5156

5257
@BeforeAll
5358
static void beforeAll() {
54-
minimalAssetString = getFileContentFromRelativePath(MINIMAL_ASSET_PATH);
55-
minimalCbmString = getFileContentFromRelativePath(MINIMAL_CBM_PATH);
59+
60+
JSON_LD_EDC.registerNamespace(VOCAB, EDC_NAMESPACE);
5661

5762
JsonSchema schemaAsset = getJsonSchema(getFileContentFromRelativePath(ASSET_SCHEMA_PATH));
58-
assetValidator = new AssetJsonSchemaValidator().getValidator(schemaAsset, JSON_LD);
63+
assetValidator = new AssetJsonSchemaValidator().getValidator(schemaAsset, JSON_LD_EDC);
5964

6065
JsonSchema schemaDataset = getJsonSchema(getFileContentFromRelativePath(DATASET_SCHEMA_PATH));
61-
datasetValidator = new DatasetJsonSchemaValidator().getValidator(schemaDataset, JSON_LD);
66+
datasetValidator = new DatasetJsonSchemaValidator().getValidator(schemaDataset, JSON_LD_EDC);
6267
}
6368

6469
@Test
6570
void transformCbmToAsset() {
71+
String minimalCbmString = getFileContentFromRelativePath(MINIMAL_CBM_PATH);
6672
JsonObject cbmJsonObject = Json.createReader(new StringReader(minimalCbmString)).readObject();
6773

68-
JsonObject cbmExpandedJsonObject = JSON_LD.expand(cbmJsonObject).getContent();
74+
JsonObject cbmExpandedJsonObject = JSON_LD_EMPTY.expand(cbmJsonObject).getContent();
75+
JsonObject assetTransformedJsonObject = CbmJsonObjectToAssetJsonObjectTransformer.transformCbmToAssetJsonObject(MINIMAL_ID, cbmExpandedJsonObject);
76+
Assertions.assertNotNull(assetTransformedJsonObject);
77+
78+
JsonObject assetTransformedCompactedJsonObject = JSON_LD_EMPTY.compact(assetTransformedJsonObject).getContent();
79+
String assetJsonTransformedString = assetTransformedCompactedJsonObject.toString();
80+
Assertions.assertNotNull(assetJsonTransformedString);
81+
82+
var result = assetValidator.validate(assetTransformedJsonObject);
83+
assertThat(result).isSucceeded();
84+
}
85+
86+
@Test
87+
void transformFullCbmToAsset() {
88+
String fullCbmString = getFileContentFromRelativePath(FULL_CBM_PATH);
89+
JsonObject cbmJsonObject = Json.createReader(new StringReader(fullCbmString)).readObject();
90+
91+
JsonObject cbmExpandedJsonObject = JSON_LD_EMPTY.expand(cbmJsonObject).getContent();
6992
JsonObject assetTransformedJsonObject = CbmJsonObjectToAssetJsonObjectTransformer.transformCbmToAssetJsonObject(MINIMAL_ID, cbmExpandedJsonObject);
7093
Assertions.assertNotNull(assetTransformedJsonObject);
7194

72-
JsonObject assetTransformedCompactedJsonObject = JSON_LD.compact(assetTransformedJsonObject).getContent();
95+
JsonObject assetTransformedCompactedJsonObject = JSON_LD_EMPTY.compact(assetTransformedJsonObject).getContent();
7396
String assetJsonTransformedString = assetTransformedCompactedJsonObject.toString();
7497
Assertions.assertNotNull(assetJsonTransformedString);
7598

@@ -79,17 +102,39 @@ void transformCbmToAsset() {
79102

80103
@Test
81104
void transformAssetToCbm() {
105+
String minimalAssetString = getFileContentFromRelativePath(MINIMAL_ASSET_PATH);
106+
82107
JsonObject assetJsonObject = Json.createReader(new StringReader(minimalAssetString)).readObject();
108+
JsonObject assetExpandedJsonObject = JSON_LD_EMPTY.expand(assetJsonObject).getContent();
109+
110+
JsonObject assetCompactedJsonObject = JSON_LD_EMPTY.compact(assetExpandedJsonObject).getContent();
111+
JsonObject cbmTransformedJsonObject = AssetJsonObjectToCbmJsonObjectTransformer.transformAssetToCbmJsonObject(MINIMAL_ID, assetCompactedJsonObject);
112+
Assertions.assertNotNull(cbmTransformedJsonObject);
113+
114+
String cbmJsonTransformedString = cbmTransformedJsonObject.toString();
115+
Assertions.assertNotNull(cbmJsonTransformedString);
116+
117+
JsonObject cbmTransformedCompactedJsonObject = JSON_LD_EDC.compact(cbmTransformedJsonObject).getContent();
118+
var result = datasetValidator.validate(cbmTransformedCompactedJsonObject);
119+
assertThat(result).isSucceeded();
120+
}
121+
122+
@Test
123+
void transformFullAssetToCbm() {
124+
String fullAssetString = getFileContentFromRelativePath(FULL_ASSET_PATH);
125+
126+
JsonObject assetJsonObject = Json.createReader(new StringReader(fullAssetString)).readObject();
127+
JsonObject assetExpandedJsonObject = JSON_LD_EMPTY.expand(assetJsonObject).getContent();
83128

84-
JsonObject assetExpandedJsonObject = JSON_LD.expand(assetJsonObject).getContent();
85-
JsonObject cbmTransformedJsonObject = AssetJsonObjectToCbmJsonObjectTransformer.transformAssetToCbmJsonObject(MINIMAL_ID, assetExpandedJsonObject);
129+
JsonObject assetCompactedJsonObject = JSON_LD_EMPTY.compact(assetExpandedJsonObject).getContent();
130+
JsonObject cbmTransformedJsonObject = AssetJsonObjectToCbmJsonObjectTransformer.transformAssetToCbmJsonObject(MINIMAL_ID, assetCompactedJsonObject);
86131
Assertions.assertNotNull(cbmTransformedJsonObject);
87132

88-
JsonObject cbmTransformedCompactedJsonObject = JSON_LD.compact(cbmTransformedJsonObject).getContent();
89-
String cbmJsonTransformedString = cbmTransformedCompactedJsonObject.toString();
133+
String cbmJsonTransformedString = cbmTransformedJsonObject.toString();
90134
Assertions.assertNotNull(cbmJsonTransformedString);
91135

92-
var result = datasetValidator.validate(cbmTransformedJsonObject);
136+
JsonObject cbmTransformedCompactedJsonObject = JSON_LD_EDC.compact(cbmTransformedJsonObject).getContent();
137+
var result = datasetValidator.validate(cbmTransformedCompactedJsonObject);
93138
assertThat(result).isSucceeded();
94139
}
95140

‎providers/provider/src/test/resources/full-cbm-dataset.json‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
2+
{
3+
"@id": "datasetFull",
4+
"@type": "dcat:Dataset",
5+
"dct:title": "Air quality in Torrent for 2021",
6+
"dct:description": "Tabular data containing a list of the daily measurements of the Torrent sensor in 2021",
7+
"dcat:keyword": ["POLLUTION", "AIR"],
8+
"contenttype": "text/csv",
9+
"dct:type": "Dataset",
10+
"dct:issued": "2022-01-22T13:04:33+00:00",
11+
"dct:modified": "2022-01-25T23:14:55+00:00",
12+
"dct:accrualPeriodicity": "annual",
13+
"dcat:temporalResolution": "P1D",
14+
"dct:temporal": {
15+
"dcat:startDate": "2021-01-01T00:00:00+00:00",
16+
"dcat:endDate": "2021-12-01T00:00:00+00:00"
17+
},
18+
"dcat:spatialResolutionInMeters": 69000.0,
19+
"dct:spatial": "POLYGON (( -0.4381504467680264 39.4105711824846 , -0.4568295983091736 39.44940414433466 , -0.6171900219935075 39.4442153805213 , -0.5905964013590401 39.39186506960127 , -0.5055720209872985 39.3881264008169 , -0.4381504467680264 39.41057118248469 ))",
20+
"dct:accessRights": "http://publications.europa.eu/resource/authority/access-right/PUBLIC",
21+
"dct:rights": "Data property of Ayto, Torrent.",
22+
"dct:license": "https://opendatacommons.org/licenses/odbl/1.0/",
23+
"schema:additionalProperty": {"domainSpecificField": 123},
24+
"cbm:hasSample": ["airQualityTorrentSample"],
25+
"dcat:inSeries": "torrentAirQualitySeries",
26+
"dcat:distribution": [
27+
{
28+
"@type": "dcat:Distribution",
29+
"dct:format": {
30+
"@id": "HttpData"
31+
},
32+
"dcat:accessURL": "http://datosabiertos.torrent.es/dataset/b6623499-2adc-4760-8392-784413d8fb41/resource/0146797f-a81c-452e-9129-8f8a11683656/download/ODMECalidadDelAirePorDias2021.csv",
33+
"proxyPath": "true",
34+
"authKey": "x-api-key",
35+
"authCode": "slq24sGHDw7c",
36+
"dcat:byteSize": 11005,
37+
"cbm:hasDataDictionary": {
38+
"@type": "cbm:DataDictionary",
39+
"cbm:hasField": [
40+
{
41+
"@type": "cbm:Field",
42+
"schema:name": "ANY",
43+
"dct:title": "Year",
44+
"cbm:datatype": "integer",
45+
"dct:description": "Year of the data accrual",
46+
"cbm:sampleValue": "2021"
47+
},
48+
{
49+
"@type": "cbm:Field",
50+
"schema:name": "FECHA",
51+
"dct:title": "Date",
52+
"cbm:datatype": "date",
53+
"dct:description": "Date of the data accrual",
54+
"cbm:sampleValue": "1/1/2021"
55+
},
56+
{
57+
"@type": "cbm:Field",
58+
"schema:name": "O3",
59+
"dct:title": "Ozone",
60+
"cbm:datatype": "integer",
61+
"dct:description": "Microgramms of ozone per cubic meter",
62+
"cbm:sampleValue": "58"
63+
}
64+
]
65+
}
66+
}
67+
],
68+
"@context": {
69+
"dcat": "http://www.w3.org/ns/dcat#",
70+
"dct": "http://purl.org/dc/terms/",
71+
"cbm": "https://w3id.org/cbm/v0.0.1/ns/",
72+
"schema": "https://schema.org/",
73+
"@vocab": "https://w3id.org/edc/v0.0.1/ns/"
74+
}
75+
}

0 commit comments

Comments
 (0)