SpringCloud/spring-cloud-function
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-1140 Add data masking capabilities for JSON logging

Browse Source 
Resolves #1140
This commit is contained in:
Oleg Zhurakousky
2024-04-30 15:20:56 +02:00
parent 59fe298b67
commit c0f4cba30d
5 changed files with 454 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
spring-cloud-function-context/src/main/java/org/springframework/cloud/function/utils/JsonMasker.java Normal file
View File

@@ -0,0 +1,140 @@
/*
* Copyright 2024-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.function.utils;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.cloud.function.json.JacksonMapper;
import org.springframework.cloud.function.json.JsonMapper;
import org.springframework.util.ClassUtils;
/**
* @author Oleg Zhurakousky
*/
public final class JsonMasker {
private static final Log logger = LogFactory.getLog(JsonMasker.class);
private static JsonMasker jsonMasker;
private final JacksonMapper mapper;
private final Set<String> keysToMask;
private JsonMasker() {
this.keysToMask = loadKeys();
this.mapper = new JacksonMapper(new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT));
}
public synchronized static JsonMasker INSTANCE() {
if (jsonMasker == null) {
jsonMasker = new JsonMasker();
}
return jsonMasker;
}
public synchronized static JsonMasker INSTANCE(Set<String> keysToMask) {
INSTANCE().addKeys(keysToMask);
return jsonMasker;
}
public String[] getKeysToMask() {
return keysToMask.toArray(new String[0]);
}
public String mask(Object json) {
if (!JsonMapper.isJsonString(json)) {
return (String) json;
}
Object map = this.mapper.fromJson(json, Object.class);
return this.iterate(map);
}
@SuppressWarnings({ "unchecked" })
private String iterate(Object json) {
if (json instanceof Collection arrayValue) {
for (Object element : arrayValue) {
if (element instanceof Map mapElement) {
for (Map.Entry<String, Object> entry : ((Map<String, Object>) mapElement).entrySet()) {
this.doMask(entry.getKey(), entry);
}
}
}
}
else if (json instanceof Map mapElement) {
for (Map.Entry<String, Object> entry : ((Map<String, Object>) mapElement).entrySet()) {
this.doMask(entry.getKey(), entry);
}
}
return new String(this.mapper.toJson(json), StandardCharsets.UTF_8);
}
private void doMask(String key, Map.Entry<String, Object> entry) {
if (this.keysToMask.contains(key)) {
entry.setValue("*******");
}
else if (entry.getValue() instanceof Map) {
this.iterate(entry.getValue());
}
else if (entry.getValue() instanceof Collection) {
this.iterate(entry.getValue());
}
}
private static Set<String> loadKeys() {
Set<String> finalKeysToMask = new TreeSet<>();
try {
Enumeration<URL> resources = ClassUtils.getDefaultClassLoader().getResources("META-INF/mask.keys");
while (resources.hasMoreElements()) {
URI uri = resources.nextElement().toURI();
List<String> lines = Files.readAllLines(Path.of(uri));
for (String line : lines) {
// need to split in case if delimited
String[] keys = line.split(",");
for (int i = 0; i < keys.length; i++) {
finalKeysToMask.add(keys[i].trim());
}
}
}
}
catch (Exception e) {
logger.warn("Failed to load keys to mask. No keys will be masked", e);
}
return finalKeysToMask;
}
private void addKeys(Set<String> keys) {
this.keysToMask.addAll(keys);
}
}

279
spring-cloud-function-context/src/test/java/org/springframework/cloud/function/utils/JsonMaskerTests.java Normal file
View File

@@ -0,0 +1,279 @@
/*
* Copyright 2024-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.function.utils;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import org.assertj.core.util.Arrays;
import org.junit.jupiter.api.Test;
import org.springframework.cloud.function.json.JacksonMapper;
import org.springframework.util.ReflectionUtils;
import static org.assertj.core.api.Assertions.assertThat;
public class JsonMaskerTests {
private String event = "{\n"
+ " \"Records\": [\n"
+ " {\n"
+ " \"eventID\": \"f07f8ca4b0b26cb9c4e5e77e69f274ee\",\n"
+ " \"eventName\": \"INSERT\",\n"
+ " \"eventVersion\": \"1.1\",\n"
+ " \"eventSource\": \"aws:dynamodb\",\n"
+ " \"awsRegion\": \"us-east-1\",\n"
+ " \"userIdentity\":{\n"
+ " \"type\":\"Service\",\n"
+ " \"principalId\":\"dynamodb.amazonaws.com\"\n"
+ " },\n"
+ " \"dynamodb\": {\n"
+ " \"ApproximateCreationDateTime\": 1.684934517E9,\n"
+ " \"Keys\": {\n"
+ " \"val\": {\n"
+ " \"S\": \"data\"\n"
+ " },\n"
+ " \"key\": {\n"
+ " \"S\": \"binary\"\n"
+ " }\n"
+ " },\n"
+ " \"NewImage\": {\n"
+ " \"val\": {\n"
+ " \"S\": \"data\"\n"
+ " },\n"
+ " \"asdf1\": {\n"
+ " \"B\": \"AAEqQQ==\"\n"
+ " },\n"
+ " \"asdf2\": {\n"
+ " \"BS\": [\n"
+ " \"AAEqQQ==\",\n"
+ " \"QSoBAA==\"\n"
+ " ]\n"
+ " },\n"
+ " \"key\": {\n"
+ " \"S\": \"binary\"\n"
+ " }\n"
+ " },\n"
+ " \"SequenceNumber\": \"1405400000000002063282832\",\n"
+ " \"SizeBytes\": 54,\n"
+ " \"StreamViewType\": \"NEW_AND_OLD_IMAGES\"\n"
+ " },\n"
+ " \"eventSourceARN\": \"arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000\"\n"
+ " },\n"
+ " {\n"
+ " \"eventID\": \"f07f8ca4b0b26cb9c4e5e77e42f274ee\",\n"
+ " \"eventName\": \"INSERT\",\n"
+ " \"eventVersion\": \"1.1\",\n"
+ " \"eventSource\": \"aws:dynamodb\",\n"
+ " \"awsRegion\": \"us-east-1\",\n"
+ " \"dynamodb\": {\n"
+ " \"ApproximateCreationDateTime\": 1480642020,\n"
+ " \"Keys\": {\n"
+ " \"val\": {\n"
+ " \"S\": \"data\"\n"
+ " },\n"
+ " \"key\": {\n"
+ " \"S\": \"binary\"\n"
+ " }\n"
+ " },\n"
+ " \"NewImage\": {\n"
+ " \"val\": {\n"
+ " \"S\": \"data\"\n"
+ " },\n"
+ " \"asdf1\": {\n"
+ " \"B\": \"AAEqQQ==\"\n"
+ " },\n"
+ " \"b2\": {\n"
+ " \"B\": \"test\"\n"
+ " },\n"
+ " \"asdf2\": {\n"
+ " \"BS\": [\n"
+ " \"AAEqQQ==\",\n"
+ " \"QSoBAA==\",\n"
+ " \"AAEqQQ==\"\n"
+ " ]\n"
+ " },\n"
+ " \"key\": {\n"
+ " \"S\": \"binary\"\n"
+ " },\n"
+ " \"Binary\": {\n"
+ " \"B\": \"AAEqQQ==\"\n"
+ " },\n"
+ " \"Boolean\": {\n"
+ " \"BOOL\": true\n"
+ " },\n"
+ " \"BinarySet\": {\n"
+ " \"BS\": [\n"
+ " \"AAEqQQ==\",\n"
+ " \"AAEqQQ==\"\n"
+ " ]\n"
+ " },\n"
+ " \"List\": {\n"
+ " \"L\": [\n"
+ " {\n"
+ " \"S\": \"Cookies\"\n"
+ " },\n"
+ " {\n"
+ " \"S\": \"Coffee\"\n"
+ " },\n"
+ " {\n"
+ " \"N\": \"3.14159\"\n"
+ " }\n"
+ " ]\n"
+ " },\n"
+ " \"Map\": {\n"
+ " \"M\": {\n"
+ " \"Name\": {\n"
+ " \"S\": \"Joe\"\n"
+ " },\n"
+ " \"Age\": {\n"
+ " \"N\": \"35\"\n"
+ " }\n"
+ " }\n"
+ " },\n"
+ " \"FloatNumber\": {\n"
+ " \"N\": \"123.45\"\n"
+ " },\n"
+ " \"IntegerNumber\": {\n"
+ " \"N\": \"123\"\n"
+ " },\n"
+ " \"NumberSet\": {\n"
+ " \"NS\": [\n"
+ " \"1234\",\n"
+ " \"567.8\"\n"
+ " ]\n"
+ " },\n"
+ " \"Null\": {\n"
+ " \"NULL\": true\n"
+ " },\n"
+ " \"String\": {\n"
+ " \"S\": \"Hello\"\n"
+ " },\n"
+ " \"StringSet\": {\n"
+ " \"SS\": [\n"
+ " \"Giraffe\",\n"
+ " \"Zebra\"\n"
+ " ]\n"
+ " },\n"
+ " \"EmptyStringSet\": {\n"
+ " \"SS\": []\n"
+ " }\n"
+ " },\n"
+ " \"SequenceNumber\": \"1405400000000002063282832\",\n"
+ " \"SizeBytes\": 54,\n"
+ " \"StreamViewType\": \"NEW_AND_OLD_IMAGES\"\n"
+ " },\n"
+ " \"eventSourceARN\": \"arn:aws:dynamodb:us-east-1:123456789012:table/Example-Table/stream/2016-12-01T00:00:00.000\"\n"
+ " }\n"
+ " ]\n"
+ "}";
private List<String> maskedKeys = new ArrayList<>();
@Test
public void validateMasking() throws Exception {
JacksonMapper mapper = new JacksonMapper(new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT));
Map<Object, Object> map = mapper.fromJson(event, Map.class);
JsonMasker masker = JsonMasker.INSTANCE();
String[] keysToMask = masker.getKeysToMask();
assertThat(keysToMask).contains("eventSourceARN", "asdf1", "SS");
String maskedJson = masker.mask(event);
System.out.println(maskedJson);
map = mapper.fromJson(maskedJson, Map.class);
this.iterate(map, Arrays.asList(keysToMask));
assertThat(maskedKeys.size()).isEqualTo(6);
assertThat(maskedKeys.get(0)).isEqualTo("asdf1");
assertThat(maskedKeys.get(1)).isEqualTo("eventSourceARN");
assertThat(maskedKeys.get(2)).isEqualTo("asdf1");
assertThat(maskedKeys.get(3)).isEqualTo("SS");
assertThat(maskedKeys.get(4)).isEqualTo("SS");
assertThat(maskedKeys.get(5)).isEqualTo("eventSourceARN");
Field jsonMaskerField = ReflectionUtils.findField(JsonMasker.class, "jsonMasker");
jsonMaskerField.setAccessible(true);
jsonMaskerField.set(masker, null);
}
@Test
public void validateMaskingWithAdditionalKeys() throws Exception {
JacksonMapper mapper = new JacksonMapper(new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT));
Map<Object, Object> map = mapper.fromJson(event, Map.class);
JsonMasker masker = JsonMasker.INSTANCE(Set.of("foo", "bar"));
String[] keysToMask = masker.getKeysToMask();
assertThat(keysToMask).contains("eventSourceARN", "asdf1", "SS", "foo", "bar");
String maskedJson = masker.mask(event);
System.out.println(maskedJson);
map = mapper.fromJson(maskedJson, Map.class);
this.iterate(map, Arrays.asList(keysToMask));
assertThat(maskedKeys.size()).isEqualTo(6);
assertThat(maskedKeys.get(0)).isEqualTo("asdf1");
assertThat(maskedKeys.get(1)).isEqualTo("eventSourceARN");
assertThat(maskedKeys.get(2)).isEqualTo("asdf1");
assertThat(maskedKeys.get(3)).isEqualTo("SS");
assertThat(maskedKeys.get(4)).isEqualTo("SS");
assertThat(maskedKeys.get(5)).isEqualTo("eventSourceARN");
Field jsonMaskerField = ReflectionUtils.findField(JsonMasker.class, "jsonMasker");
jsonMaskerField.setAccessible(true);
jsonMaskerField.set(masker, null);
}
@SuppressWarnings({ "unchecked", "rawtypes" })
private void iterate(Object json, List keysToMask) {
if (json instanceof Collection arrayValue) {
for (Object element : arrayValue) {
if (element instanceof Map mapElement) {
for (Map.Entry<String, Object> entry : ((Map<String, Object>) mapElement).entrySet()) {
this.doMask(entry.getKey(), entry, keysToMask);
}
}
}
}
else if (json instanceof Map mapElement) {
for (Map.Entry<String, Object> entry : ((Map<String, Object>) mapElement).entrySet()) {
this.doMask(entry.getKey(), entry, keysToMask);
}
}
}
@SuppressWarnings("rawtypes")
private void doMask(String key, Map.Entry<String, Object> entry, List keysToMask) {
if (keysToMask.contains(key)) {
System.out.println("Masked: " + entry.getKey());
maskedKeys.add(key);
}
else if (entry.getValue() instanceof Map) {
this.iterate(entry.getValue(), keysToMask);
}
else if (entry.getValue() instanceof Collection) {
this.iterate(entry.getValue(), keysToMask);
}
}
}

2
spring-cloud-function-context/src/test/resources/META-INF/mask.keys Normal file
View File

@@ -0,0 +1,2 @@
eventSourceARN
asdf1, SS