diff --git a/docs/modules/ROOT/pages/spring-cloud-function/programming-model.adoc b/docs/modules/ROOT/pages/spring-cloud-function/programming-model.adoc index 51cf7e016..38e4b59fe 100644 --- a/docs/modules/ROOT/pages/spring-cloud-function/programming-model.adoc +++ b/docs/modules/ROOT/pages/spring-cloud-function/programming-model.adoc @@ -715,3 +715,28 @@ Spring Cloud Function will scan for implementations of `Function`, `Consumer` an feature you can write functions that have no dependencies on Spring - not even the `@Component` annotation is needed. If you want to use a different package, you can set `spring.cloud.function.scan.packages`. You can also use `spring.cloud.function.scan.enabled=false` to switch off the scan completely. + +== Data Masking + +A typical application comes with several levels of logging. Certain cloud/serverless platforms may include sensitive data in the packets that are being logged for everyone to see. +While it is the responsibility of individual developer to inspect the data that is being logged, so logging comes from the framework itself, so since version 4.1 we have introduced `JsonMasker` to initially help with masking sensitive data in AWS Lambda payloads. However, the `JsonMasker` is generic and is available to any module. At the moment it will only work with structured data such as JSON. All you need is to specify the keys you want to mask and it will take care of the rest. +Keys should be specified in the file `META-INF/mask.keys`. The format of the file is very simple where you can delimit several keys by commas or new line or both. + +Here is the example of the contents of such file: + +---- +eventSourceARN +asdf1, SS +---- + +Here you see three keys are defined +Once such file exists, the JsonMasker will use it to mask values of the keys specified. + +And here is the sample code that shows the usage + +---- +private final static JsonMasker masker = JsonMasker.INSTANCE(); +. . . + +logger.info("Received: " + masker.mask(new String(payload, StandardCharsets.UTF_8))); +---- \ No newline at end of file diff --git a/spring-cloud-function-adapters/spring-cloud-function-adapter-aws/src/main/java/org/springframework/cloud/function/adapter/aws/AWSLambdaUtils.java b/spring-cloud-function-adapters/spring-cloud-function-adapter-aws/src/main/java/org/springframework/cloud/function/adapter/aws/AWSLambdaUtils.java index 916ff31a0..4f27ad980 100644 --- a/spring-cloud-function-adapters/spring-cloud-function-adapter-aws/src/main/java/org/springframework/cloud/function/adapter/aws/AWSLambdaUtils.java +++ b/spring-cloud-function-adapters/spring-cloud-function-adapter-aws/src/main/java/org/springframework/cloud/function/adapter/aws/AWSLambdaUtils.java @@ -34,6 +34,7 @@ import reactor.core.publisher.Flux; import org.springframework.cloud.function.context.catalog.FunctionTypeUtils; import org.springframework.cloud.function.json.JsonMapper; +import org.springframework.cloud.function.utils.JsonMasker; import org.springframework.http.HttpStatus; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHeaders; @@ -67,6 +68,8 @@ public final class AWSLambdaUtils { */ public static final String AWS_CONTEXT = "aws-context"; + private final static JsonMasker masker = JsonMasker.INSTANCE(); + private AWSLambdaUtils() { } @@ -102,11 +105,15 @@ public final class AWSLambdaUtils { return generateMessage(payload, inputType, isSupplier, jsonMapper, null); } + private static String mask(String value) { + return masker.mask(value); + } + @SuppressWarnings({ "unchecked", "rawtypes" }) public static Message generateMessage(byte[] payload, Type inputType, boolean isSupplier, JsonMapper jsonMapper, Context context) { if (logger.isInfoEnabled()) { - logger.info("Received: " + new String(payload, StandardCharsets.UTF_8)); + logger.info("Received: " + mask(new String(payload, StandardCharsets.UTF_8))); } Object structMessage = jsonMapper.fromJson(payload, Object.class); diff --git a/spring-cloud-function-context/src/main/java/org/springframework/cloud/function/utils/JsonMasker.java b/spring-cloud-function-context/src/main/java/org/springframework/cloud/function/utils/JsonMasker.java new file mode 100644 index 000000000..a6a5da86d --- /dev/null +++ b/spring-cloud-function-context/src/main/java/org/springframework/cloud/function/utils/JsonMasker.java @@ -0,0 +1,140 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.function.utils; + +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.cloud.function.json.JacksonMapper; +import org.springframework.cloud.function.json.JsonMapper; +import org.springframework.util.ClassUtils; + + +/** + * @author Oleg Zhurakousky + */ +public final class JsonMasker { + + private static final Log logger = LogFactory.getLog(JsonMasker.class); + + private static JsonMasker jsonMasker; + + private final JacksonMapper mapper; + + private final Set keysToMask; + + private JsonMasker() { + this.keysToMask = loadKeys(); + this.mapper = new JacksonMapper(new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT)); + + } + + public synchronized static JsonMasker INSTANCE() { + if (jsonMasker == null) { + jsonMasker = new JsonMasker(); + } + return jsonMasker; + } + + public synchronized static JsonMasker INSTANCE(Set keysToMask) { + INSTANCE().addKeys(keysToMask); + return jsonMasker; + } + + public String[] getKeysToMask() { + return keysToMask.toArray(new String[0]); + } + + public String mask(Object json) { + if (!JsonMapper.isJsonString(json)) { + return (String) json; + } + Object map = this.mapper.fromJson(json, Object.class); + return this.iterate(map); + } + + @SuppressWarnings({ "unchecked" }) + private String iterate(Object json) { + if (json instanceof Collection arrayValue) { + for (Object element : arrayValue) { + if (element instanceof Map mapElement) { + for (Map.Entry entry : ((Map) mapElement).entrySet()) { + this.doMask(entry.getKey(), entry); + } + } + } + } + else if (json instanceof Map mapElement) { + for (Map.Entry entry : ((Map) mapElement).entrySet()) { + this.doMask(entry.getKey(), entry); + } + } + return new String(this.mapper.toJson(json), StandardCharsets.UTF_8); + } + + private void doMask(String key, Map.Entry entry) { + if (this.keysToMask.contains(key)) { + entry.setValue("*******"); + } + else if (entry.getValue() instanceof Map) { + this.iterate(entry.getValue()); + } + else if (entry.getValue() instanceof Collection) { + this.iterate(entry.getValue()); + } + } + + private static Set loadKeys() { + Set finalKeysToMask = new TreeSet<>(); + try { + Enumeration resources = ClassUtils.getDefaultClassLoader().getResources("META-INF/mask.keys"); + while (resources.hasMoreElements()) { + URI uri = resources.nextElement().toURI(); + List lines = Files.readAllLines(Path.of(uri)); + for (String line : lines) { + // need to split in case if delimited + String[] keys = line.split(","); + for (int i = 0; i < keys.length; i++) { + finalKeysToMask.add(keys[i].trim()); + } + } + } + } + catch (Exception e) { + logger.warn("Failed to load keys to mask. No keys will be masked", e); + } + return finalKeysToMask; + } + + private void addKeys(Set keys) { + this.keysToMask.addAll(keys); + } +} diff --git a/spring-cloud-function-context/src/test/java/org/springframework/cloud/function/utils/JsonMaskerTests.java b/spring-cloud-function-context/src/test/java/org/springframework/cloud/function/utils/JsonMaskerTests.java new file mode 100644 index 000000000..37c7ce0e8 --- /dev/null +++ b/spring-cloud-function-context/src/test/java/org/springframework/cloud/function/utils/JsonMaskerTests.java @@ -0,0 +1,279 @@ +/* + * Copyright 2024-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.function.utils; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import org.assertj.core.util.Arrays; +import org.junit.jupiter.api.Test; + +import org.springframework.cloud.function.json.JacksonMapper; +import org.springframework.util.ReflectionUtils; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class JsonMaskerTests { + + private String event = "{\n" + + " \"Records\": [\n" + + " {\n" + + " \"eventID\": \"f07f8ca4b0b26cb9c4e5e77e69f274ee\",\n" + + " \"eventName\": \"INSERT\",\n" + + " \"eventVersion\": \"1.1\",\n" + + " \"eventSource\": \"aws:dynamodb\",\n" + + " \"awsRegion\": \"us-east-1\",\n" + + " \"userIdentity\":{\n" + + " \"type\":\"Service\",\n" + + " \"principalId\":\"dynamodb.amazonaws.com\"\n" + + " },\n" + + " \"dynamodb\": {\n" + + " \"ApproximateCreationDateTime\": 1.684934517E9,\n" + + " \"Keys\": {\n" + + " \"val\": {\n" + + " \"S\": \"data\"\n" + + " },\n" + + " \"key\": {\n" + + " \"S\": \"binary\"\n" + + " }\n" + + " },\n" + + " \"NewImage\": {\n" + + " \"val\": {\n" + + " \"S\": \"data\"\n" + + " },\n" + + " \"asdf1\": {\n" + + " \"B\": \"AAEqQQ==\"\n" + + " },\n" + + " \"asdf2\": {\n" + + " \"BS\": [\n" + + " \"AAEqQQ==\",\n" + + " \"QSoBAA==\"\n" + + " ]\n" + + " },\n" + + " \"key\": {\n" + + " \"S\": \"binary\"\n" + + " }\n" + + " },\n" + + " \"SequenceNumber\": \"1405400000000002063282832\",\n" + + " \"SizeBytes\": 54,\n" + + " \"StreamViewType\": \"NEW_AND_OLD_IMAGES\"\n" + + " },\n" + + " \"eventSourceARN\": \"arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000\"\n" + + " },\n" + + " {\n" + + " \"eventID\": \"f07f8ca4b0b26cb9c4e5e77e42f274ee\",\n" + + " \"eventName\": \"INSERT\",\n" + + " \"eventVersion\": \"1.1\",\n" + + " \"eventSource\": \"aws:dynamodb\",\n" + + " \"awsRegion\": \"us-east-1\",\n" + + " \"dynamodb\": {\n" + + " \"ApproximateCreationDateTime\": 1480642020,\n" + + " \"Keys\": {\n" + + " \"val\": {\n" + + " \"S\": \"data\"\n" + + " },\n" + + " \"key\": {\n" + + " \"S\": \"binary\"\n" + + " }\n" + + " },\n" + + " \"NewImage\": {\n" + + " \"val\": {\n" + + " \"S\": \"data\"\n" + + " },\n" + + " \"asdf1\": {\n" + + " \"B\": \"AAEqQQ==\"\n" + + " },\n" + + " \"b2\": {\n" + + " \"B\": \"test\"\n" + + " },\n" + + " \"asdf2\": {\n" + + " \"BS\": [\n" + + " \"AAEqQQ==\",\n" + + " \"QSoBAA==\",\n" + + " \"AAEqQQ==\"\n" + + " ]\n" + + " },\n" + + " \"key\": {\n" + + " \"S\": \"binary\"\n" + + " },\n" + + " \"Binary\": {\n" + + " \"B\": \"AAEqQQ==\"\n" + + " },\n" + + " \"Boolean\": {\n" + + " \"BOOL\": true\n" + + " },\n" + + " \"BinarySet\": {\n" + + " \"BS\": [\n" + + " \"AAEqQQ==\",\n" + + " \"AAEqQQ==\"\n" + + " ]\n" + + " },\n" + + " \"List\": {\n" + + " \"L\": [\n" + + " {\n" + + " \"S\": \"Cookies\"\n" + + " },\n" + + " {\n" + + " \"S\": \"Coffee\"\n" + + " },\n" + + " {\n" + + " \"N\": \"3.14159\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " \"Map\": {\n" + + " \"M\": {\n" + + " \"Name\": {\n" + + " \"S\": \"Joe\"\n" + + " },\n" + + " \"Age\": {\n" + + " \"N\": \"35\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"FloatNumber\": {\n" + + " \"N\": \"123.45\"\n" + + " },\n" + + " \"IntegerNumber\": {\n" + + " \"N\": \"123\"\n" + + " },\n" + + " \"NumberSet\": {\n" + + " \"NS\": [\n" + + " \"1234\",\n" + + " \"567.8\"\n" + + " ]\n" + + " },\n" + + " \"Null\": {\n" + + " \"NULL\": true\n" + + " },\n" + + " \"String\": {\n" + + " \"S\": \"Hello\"\n" + + " },\n" + + " \"StringSet\": {\n" + + " \"SS\": [\n" + + " \"Giraffe\",\n" + + " \"Zebra\"\n" + + " ]\n" + + " },\n" + + " \"EmptyStringSet\": {\n" + + " \"SS\": []\n" + + " }\n" + + " },\n" + + " \"SequenceNumber\": \"1405400000000002063282832\",\n" + + " \"SizeBytes\": 54,\n" + + " \"StreamViewType\": \"NEW_AND_OLD_IMAGES\"\n" + + " },\n" + + " \"eventSourceARN\": \"arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000\"\n" + + " }\n" + + " ]\n" + + "}"; + + private List maskedKeys = new ArrayList<>(); + + @Test + public void validateMasking() throws Exception { + JacksonMapper mapper = new JacksonMapper(new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT)); + Map map = mapper.fromJson(event, Map.class); + + JsonMasker masker = JsonMasker.INSTANCE(); + String[] keysToMask = masker.getKeysToMask(); + assertThat(keysToMask).contains("eventSourceARN", "asdf1", "SS"); + + String maskedJson = masker.mask(event); + System.out.println(maskedJson); + map = mapper.fromJson(maskedJson, Map.class); + + this.iterate(map, Arrays.asList(keysToMask)); + assertThat(maskedKeys.size()).isEqualTo(6); + assertThat(maskedKeys.get(0)).isEqualTo("asdf1"); + assertThat(maskedKeys.get(1)).isEqualTo("eventSourceARN"); + assertThat(maskedKeys.get(2)).isEqualTo("asdf1"); + assertThat(maskedKeys.get(3)).isEqualTo("SS"); + assertThat(maskedKeys.get(4)).isEqualTo("SS"); + assertThat(maskedKeys.get(5)).isEqualTo("eventSourceARN"); + + Field jsonMaskerField = ReflectionUtils.findField(JsonMasker.class, "jsonMasker"); + jsonMaskerField.setAccessible(true); + jsonMaskerField.set(masker, null); + } + + @Test + public void validateMaskingWithAdditionalKeys() throws Exception { + JacksonMapper mapper = new JacksonMapper(new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT)); + Map map = mapper.fromJson(event, Map.class); + + JsonMasker masker = JsonMasker.INSTANCE(Set.of("foo", "bar")); + String[] keysToMask = masker.getKeysToMask(); + assertThat(keysToMask).contains("eventSourceARN", "asdf1", "SS", "foo", "bar"); + + String maskedJson = masker.mask(event); + System.out.println(maskedJson); + map = mapper.fromJson(maskedJson, Map.class); + + this.iterate(map, Arrays.asList(keysToMask)); + assertThat(maskedKeys.size()).isEqualTo(6); + assertThat(maskedKeys.get(0)).isEqualTo("asdf1"); + assertThat(maskedKeys.get(1)).isEqualTo("eventSourceARN"); + assertThat(maskedKeys.get(2)).isEqualTo("asdf1"); + assertThat(maskedKeys.get(3)).isEqualTo("SS"); + assertThat(maskedKeys.get(4)).isEqualTo("SS"); + assertThat(maskedKeys.get(5)).isEqualTo("eventSourceARN"); + + Field jsonMaskerField = ReflectionUtils.findField(JsonMasker.class, "jsonMasker"); + jsonMaskerField.setAccessible(true); + jsonMaskerField.set(masker, null); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private void iterate(Object json, List keysToMask) { + if (json instanceof Collection arrayValue) { + for (Object element : arrayValue) { + if (element instanceof Map mapElement) { + for (Map.Entry entry : ((Map) mapElement).entrySet()) { + this.doMask(entry.getKey(), entry, keysToMask); + } + } + } + } + else if (json instanceof Map mapElement) { + for (Map.Entry entry : ((Map) mapElement).entrySet()) { + this.doMask(entry.getKey(), entry, keysToMask); + } + } + } + + @SuppressWarnings("rawtypes") + private void doMask(String key, Map.Entry entry, List keysToMask) { + if (keysToMask.contains(key)) { + System.out.println("Masked: " + entry.getKey()); + maskedKeys.add(key); + } + else if (entry.getValue() instanceof Map) { + this.iterate(entry.getValue(), keysToMask); + } + else if (entry.getValue() instanceof Collection) { + this.iterate(entry.getValue(), keysToMask); + } + } +} diff --git a/spring-cloud-function-context/src/test/resources/META-INF/mask.keys b/spring-cloud-function-context/src/test/resources/META-INF/mask.keys new file mode 100644 index 000000000..fadb6a069 --- /dev/null +++ b/spring-cloud-function-context/src/test/resources/META-INF/mask.keys @@ -0,0 +1,2 @@ +eventSourceARN +asdf1, SS