Account for !--- doc splitter for properties validation

headless-services/commons/java-properties/src/main/java/org/springframework/ide/vscode/java/properties/parser/PropertiesAst.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2016 Pivotal, Inc.
+ * Copyright (c) 2016, 2024 Pivotal, Inc.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -11,8 +11,6 @@
 package org.springframework.ide.vscode.java.properties.parser;
 
 import java.util.List;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 /**
  * Very basic AST for Java Properties to keep comments and key value pairs
@@ -36,21 +34,12 @@ public final class PropertiesAst {
 		return nodes;
 	}
 
-	public List<Node> getNodes(Predicate<Node> test) {
+	public List<KeyValuePair> getPropertyValuePairs() {
-		return nodes.stream().filter(test).collect(Collectors.toList());
+		return nodes.stream().filter(KeyValuePair.class::isInstance).map(KeyValuePair.class::cast).toList();
 	}
 	
-	/**
+	public List<Comment> getComments() {
-	 * Retrieves AST nodes of specific type
+		return nodes.stream().filter(Comment.class::isInstance).map(Comment.class::cast).toList();
-	 * @param clazz Type of AST nodes
-	 * @return List of AST nodes of specific type sorted by line number
-	 */
-	@SuppressWarnings("unchecked")
-	public <T> List<T> getNodes(Class<T> clazz) {
-		List<Node> l = nodes.stream().filter(line -> {
-			return clazz.isAssignableFrom(line.getClass());
-		}).collect(Collectors.toList());
-		return (List<T>) l;
 	}
 	
 	/**

headless-services/commons/java-properties/src/test/java/org/springframework/ide/vscode/java/properties/parser/test/PropertiesAntlrParserTest.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2016 Pivotal, Inc.
+ * Copyright (c) 2016, 2024 Pivotal, Inc.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -35,7 +35,7 @@ public class PropertiesAntlrParserTest {
 		assertTrue(results.problems.isEmpty());
 		assertEquals(1, results.ast.getAllNodes().size());
 		
-		List<Comment> commentLines = results.ast.getNodes(Comment.class);
+		List<Comment> commentLines = results.ast.getComments();
 		assertEquals(1, commentLines.size());
 		
 		Comment comment = commentLines.get(0);
@@ -52,7 +52,7 @@ public class PropertiesAntlrParserTest {
 		assertTrue(results.problems.isEmpty());
 		assertEquals(1, results.ast.getAllNodes().size());
 		
-		List<KeyValuePair> propertyLines = results.ast.getNodes(KeyValuePair.class);
+		List<KeyValuePair> propertyLines = results.ast.getPropertyValuePairs();
 		assertEquals(1, propertyLines.size());
 		
 		KeyValuePair line = propertyLines.get(0);
@@ -197,7 +197,7 @@ public class PropertiesAntlrParserTest {
 		assertTrue(results.problems.isEmpty());
 		// One property line recorded. With key and empty value 
 		assertEquals(3, results.ast.getAllNodes().size());
-		List<KeyValuePair> lines = results.ast.getNodes(KeyValuePair.class);
+		List<KeyValuePair> lines = results.ast.getPropertyValuePairs();
 		assertEquals(3, lines.size());
 		
 		// Test valid part

headless-services/commons/java-properties/src/test/java/org/springframework/ide/vscode/java/properties/parser/test/PropertiesAstTest.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2016-2017 Pivotal, Inc.
+ * Copyright (c) 2016, 2024 Pivotal, Inc.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -36,8 +36,8 @@ public class PropertiesAstTest {
 		assertTrue(results.syntaxErrors.isEmpty());
 		assertTrue(results.problems.isEmpty());
 		assertEquals(4, results.ast.getAllNodes().size());
-		assertEquals(1, results.ast.getNodes(Comment.class).size());
+		assertEquals(1, results.ast.getComments().size());
-		assertEquals(3, results.ast.getNodes(EmptyLine.class).size());
+//		assertEquals(3, results.ast.getNodes(EmptyLine.class).size());
 	}
 	
 	@Test
@@ -46,8 +46,8 @@ public class PropertiesAstTest {
 		assertTrue(results.syntaxErrors.isEmpty());
 		assertTrue(results.problems.isEmpty());
 		assertEquals(5, results.ast.getAllNodes().size());
-		assertEquals(1, results.ast.getNodes(Comment.class).size());
+		assertEquals(1, results.ast.getComments().size());
-		assertEquals(4, results.ast.getNodes(EmptyLine.class).size());
+//		assertEquals(4, results.ast.getNodes(EmptyLine.class).size());
 	}
 	
 	@Test
@@ -56,26 +56,26 @@ public class PropertiesAstTest {
 		assertTrue(results.syntaxErrors.isEmpty());
 		assertTrue(results.problems.isEmpty());
 		assertEquals(5, results.ast.getAllNodes().size());
-		assertEquals(1, results.ast.getNodes(Comment.class).size());
+		assertEquals(1, results.ast.getComments().size());
-		assertEquals(1, results.ast.getNodes(KeyValuePair.class).size());		
+		assertEquals(1, results.ast.getPropertyValuePairs().size());		
-		assertEquals(3, results.ast.getNodes(EmptyLine.class).size());
+//		assertEquals(3, results.ast.getNodes(EmptyLine.class).size());
 	}
 	
 	@Test
 	public void testLines4() throws Exception {
 		ParseResults results = parser.parse("# Comment-1\n\nkey = value  1  \n# Comment-2");
 		assertEquals(4, results.ast.getAllNodes().size());
-		assertEquals(2, results.ast.getNodes(Comment.class).size());
+		assertEquals(2, results.ast.getComments().size());
-		assertEquals(1, results.ast.getNodes(KeyValuePair.class).size());		
+		assertEquals(1, results.ast.getPropertyValuePairs().size());		
-		assertEquals(1, results.ast.getNodes(EmptyLine.class).size());
+//		assertEquals(1, results.ast.getNodes(EmptyLine.class).size());
 	}
 	
 	@Test
 	public void testLines5() throws Exception {
 		ParseResults results = parser.parse("#comment\nliquibase.enabled=\n#comment");
 		assertEquals(3, results.ast.getAllNodes().size());
-		assertEquals(2, results.ast.getNodes(Comment.class).size());
+		assertEquals(2, results.ast.getComments().size());
-		assertEquals(1, results.ast.getNodes(KeyValuePair.class).size());		
+		assertEquals(1, results.ast.getPropertyValuePairs().size());		
 	}
 		
 	@Test

headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/factories/SpringFactoriesReconcileEngine.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2022 VMware, Inc.
+ * Copyright (c) 2022, 2024 VMware, Inc.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -63,7 +63,7 @@ public class SpringFactoriesReconcileEngine implements IReconcileEngine {
 			try {
 				PropertiesAst ast = parser.parse(doc.get()).ast;
 				if (ast != null) {
-					for (KeyValuePair pair : ast.getNodes(KeyValuePair.class)) {
+					for (KeyValuePair pair : ast.getPropertyValuePairs()) {
 						String key = pair.getKey().decode().trim();
 						KeyValuePairReconciler r = keyValuePairReconcilers.get(key);
 						if (r != null) {

headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/data/jpa/queries/NamedQueryPropertiesReconcileEngine.java

@@ -43,7 +43,7 @@ public class NamedQueryPropertiesReconcileEngine implements IReconcileEngine {
 			
 			AntlrParser parser = new AntlrParser();
 			ParseResults parseResults = parser.parse(doc.get());
-			for (KeyValuePair pair : parseResults.ast.getNodes(KeyValuePair.class)) {
+			for (KeyValuePair pair : parseResults.ast.getPropertyValuePairs()) {
 				Value value = pair.getValue();
 				reconciler.reconcile(value.decode(), value.getOffset(), problemCollector);
 			}

headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/data/jpa/queries/QueryPropertiesSemanticTokensHandler.java

@@ -73,7 +73,7 @@ public class QueryPropertiesSemanticTokensHandler implements SemanticTokensHandl
 				AntlrParser propertiesParser = new AntlrParser();
 				ParseResults result = propertiesParser.parse(doc.get());
 				List<SemanticTokenData> data = new ArrayList<>();
-				for (PropertiesAst.KeyValuePair node : result.ast.getNodes(PropertiesAst.KeyValuePair.class)) {
+				for (PropertiesAst.KeyValuePair node : result.ast.getPropertyValuePairs()) {
 					Value value = node.getValue();
 					if (value != null) {
 						data.addAll(tokensProvider.computeTokens(value.decode(), value.getOffset()));

headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/utils/SpringFactoriesIndexer.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2023 VMware, Inc.
+ * Copyright (c) 2023, 2024 VMware, Inc.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -52,7 +52,6 @@ import org.springframework.ide.vscode.commons.util.text.TextDocument;
 import org.springframework.ide.vscode.java.properties.antlr.parser.AntlrParser;
 import org.springframework.ide.vscode.java.properties.parser.PropertiesAst;
 import org.springframework.ide.vscode.java.properties.parser.PropertiesAst.KeyValuePair;
-import org.springframework.ide.vscode.java.properties.parser.PropertiesAst.Node;
 
 import com.google.common.collect.ImmutableList;
 
@@ -109,8 +108,7 @@ public class SpringFactoriesIndexer implements SpringIndexer {
 		ImmutableList.Builder<EnhancedSymbolInformation> symbols = ImmutableList.builder();
 		PropertiesAst ast = new AntlrParser().parse(content).ast;
 		if (ast != null) {
-			for (Node n : ast.getNodes(KeyValuePair.class::isInstance)) {
+			for (KeyValuePair pair : ast.getPropertyValuePairs()) {
-				KeyValuePair pair = (KeyValuePair) n;
 				String key = pair.getKey().decode();
 				
 				if (KEYS.contains(key)) {

headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/value/ValuePropertyReferencesProvider.java

@@ -285,7 +285,7 @@ public class ValuePropertyReferencesProvider implements ReferenceProvider {
 			ParseResults parseResults = parser.parse(fileContent);
 	
 			if (parseResults != null && parseResults.ast != null) {
-				parseResults.ast.getNodes(KeyValuePair.class).forEach(pair -> {
+				parseResults.ast.getPropertyValuePairs().forEach(pair -> {
 					if (pair.getKey() != null && pair.getKey().decode().equals(propertyKey)) {
 						TextDocument doc = new TextDocument(file.toURI().toASCIIString(), null);
 						doc.setText(fileContent);

headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/properties/reconcile/SpringPropertiesReconcileEngine.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2014, 2019 Pivotal, Inc.
+ * Copyright (c) 2014, 2024 Pivotal, Inc.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -16,6 +16,7 @@ import static org.springframework.ide.vscode.boot.properties.reconcile.Applicati
 import static org.springframework.ide.vscode.boot.properties.reconcile.SpringPropertyProblem.problem;
 import static org.springframework.ide.vscode.commons.util.StringUtil.commonPrefix;
 
+import java.util.List;
 import java.util.regex.Pattern;
 
 import org.eclipse.lsp4j.TextDocumentIdentifier;
@@ -29,10 +30,10 @@ import org.springframework.ide.vscode.boot.metadata.types.Type;
 import org.springframework.ide.vscode.boot.metadata.types.TypeParser;
 import org.springframework.ide.vscode.boot.metadata.types.TypeUtil;
 import org.springframework.ide.vscode.boot.metadata.types.TypeUtilProvider;
-import org.springframework.ide.vscode.boot.properties.quickfix.DeprecatedPropertyData;
-import org.springframework.ide.vscode.boot.properties.quickfix.MissingPropertyData;
 import org.springframework.ide.vscode.boot.properties.quickfix.AppPropertiesQuickFixes;
 import org.springframework.ide.vscode.boot.properties.quickfix.CommonQuickfixes;
+import org.springframework.ide.vscode.boot.properties.quickfix.DeprecatedPropertyData;
+import org.springframework.ide.vscode.boot.properties.quickfix.MissingPropertyData;
 import org.springframework.ide.vscode.commons.languageserver.quickfix.Quickfix.QuickfixData;
 import org.springframework.ide.vscode.commons.languageserver.quickfix.QuickfixType;
 import org.springframework.ide.vscode.commons.languageserver.reconcile.IProblemCollector;
@@ -47,8 +48,8 @@ import org.springframework.ide.vscode.commons.util.text.IDocument;
 import org.springframework.ide.vscode.java.properties.antlr.parser.AntlrParser;
 import org.springframework.ide.vscode.java.properties.parser.ParseResults;
 import org.springframework.ide.vscode.java.properties.parser.Parser;
-import org.springframework.ide.vscode.java.properties.parser.PropertiesAst.KeyValuePair;
 import org.springframework.ide.vscode.java.properties.parser.PropertiesAst.Comment;
+import org.springframework.ide.vscode.java.properties.parser.PropertiesAst.KeyValuePair;
 import org.springframework.ide.vscode.java.properties.parser.PropertiesAst.Node;
 import org.springframework.ide.vscode.java.properties.parser.PropertiesFileEscapes;
 
@@ -68,9 +69,9 @@ public class SpringPropertiesReconcileEngine implements IReconcileEngine {
 	/**
 	 * Regexp that matches a ',' surrounded by whitespace, including escaped whitespace / newlines
 	 */
-	private static final Pattern COMMA = Pattern.compile(
+//	private static final Pattern COMMA = Pattern.compile(
-			"(\\s|\\\\\\s)*,(\\s|\\\\\\s)*"
+//			"(\\s|\\\\\\s)*,(\\s|\\\\\\s)*"
-	);
+//	);
 
 	private static final Pattern SPACES = Pattern.compile(
 			"(\\s|\\\\\\s)*"
@@ -109,10 +110,12 @@ public class SpringPropertiesReconcileEngine implements IReconcileEngine {
 				return;
 			}
 			
-			results.ast.getNodes(n -> n instanceof KeyValuePair || n instanceof Comment).forEach(node -> {
+			List<Node> nodes = results.ast.getAllNodes();
+			for (int i = 0; i < results.ast.getAllNodes().size(); i++) {
+				Node node = nodes.get(i);
 				try {
 					if (node instanceof Comment) {
-						if (isDocumentMarker(doc, (Comment)node)) {
+						if (isDocumentMarker(doc, nodes, i)) {
 							duplicateNameChecker.startNewSubDocument();
 						}
 					} else if (node instanceof KeyValuePair) {
@@ -144,7 +147,8 @@ public class SpringPropertiesReconcileEngine implements IReconcileEngine {
 				} catch (Exception e) {
 					log.error("", e);
 				}
-			});
+			}
+
 		} catch (Throwable e2) {
 			log.error("", e2);
 		} finally {
@@ -152,10 +156,43 @@ public class SpringPropertiesReconcileEngine implements IReconcileEngine {
 		}
 	}
 
-	private boolean isDocumentMarker(IDocument doc, Comment node) {
+	private boolean isDocumentMarker(IDocument doc, List<Node> nodes, int idx) {
-		DocumentRegion region = createRegion(doc, node).trimEnd();
+		Node node = nodes.get(idx);
-		String t = region.toString();
+		if (node instanceof Comment) {
-		return t.equals("#---") && region.isAtStartOfLine();
+			DocumentRegion region = createRegion(doc, node).trimEnd();
+			String t = region.toString();
+			if (region.isAtStartOfLine()) {
+				switch (t) {
+				case "#---":
+					// Check if next line starts with the same comment prefix. If yes then not a doc splitter
+					if (isNextLineCommentWithPrefix(doc, nodes, idx, "#")) {
+						return false;
+					}
+					return true;
+				case "!---":
+					// Check if next line starts with the same comment prefix. If yes then not a doc splitter
+					if (isNextLineCommentWithPrefix(doc, nodes, idx, "!")) {
+						return false;
+					}
+					return true;
+				}
+				
+			}
+		}
+		return false;
+	}
+	
+	private boolean isNextLineCommentWithPrefix(IDocument doc, List<Node> nodes, int idx, String prefix) {
+		if (idx < nodes.size() - 1) {
+			Node nextLine = nodes.get(idx + 1);
+			if (nextLine instanceof Comment) {
+				DocumentRegion nextRegion = createRegion(doc, nextLine).trimEnd();
+				if (nextRegion.toString().startsWith(prefix)) {
+					return true;
+				}
+			}
+		}
+		return false;
 	}
 
 	protected SpringPropertyProblem problemDeprecated(DocumentRegion region, PropertyInfo property, QuickfixType fixType) {

headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/test/ApplicationPropertiesEditorTest.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2016, 2023 Pivotal, Inc.
+ * Copyright (c) 2016, 2024 Pivotal, Inc.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -98,7 +98,7 @@ public class ApplicationPropertiesEditorTest extends AbstractPropsEditorTest {
     }
 
     @Test
-    void reconcilesWithMultiDocuments() throws Exception {
+    void reconcilesWithMultiDocuments_Hash() throws Exception {
         //See: https://github.com/spring-projects/sts4/issues/533
         
         defaultTestData();
@@ -156,6 +156,127 @@ public class ApplicationPropertiesEditorTest extends AbstractPropsEditorTest {
         );
         editor.assertProblems(/*NONE*/);
         
+        editor = newEditor(
+                "spring.application.name=frodo\n" +
+                        "spring.config.activate.on-profile=foo\n" +
+                        "server.port=8888\n" +
+                        "#---\n" +
+                        "# Something off\n" +
+                        "spring.application.name=sam\n" +
+                        "spring.config.activate.on-profile=bar\n" +
+                        "server.port=9090\n"
+        );
+        editor.assertProblems(
+                "spring.application.name|Duplicate",
+                "spring.config.activate.on-profile|Duplicate",
+                "server.port|Duplicate",
+                "spring.application.name|Duplicate",
+                "spring.config.activate.on-profile|Duplicate",
+                "server.port|Duplicate"
+        );
+
+        editor = newEditor(
+                "spring.application.name=frodo\n" +
+                        "spring.config.activate.on-profile=foo\n" +
+                        "server.port=8888\n" +
+                        "#---\n" +
+                        "! Something off\n" +
+                        "spring.application.name=sam\n" +
+                        "spring.config.activate.on-profile=bar\n" +
+                        "server.port=9090\n"
+        );
+        editor.assertProblems(/*NONE*/);
+    }
+
+    @Test
+    void reconcilesWithMultiDocuments_Exclamation() throws Exception {
+        //See: https://github.com/spring-projects/sts4/issues/1129
+        
+        defaultTestData();
+
+        //lowest bar: just disable the warning
+        Editor editor = newEditor(
+                "spring.config.activate.on-profile=foo\n" +
+                        "server.port=8888\n" +
+                        "!---\n" +
+                        "spring.config.activate.on-profile=bar\n" +
+                        "server.port=8080\n"
+        );
+        editor.assertProblems(/*NONE*/);
+
+        //better: still detect duplicates within same section
+        editor = newEditor(
+                "spring.application.name=frodo\n" +
+                        "spring.config.activate.on-profile=foo\n" +
+                        "server.port=8888\n" +
+                        "spring.application.name=frodo\n" +
+                        "!---\n" +
+                        "spring.config.activate.on-profile=bar\n" +
+                        "server.port=8080\n" +
+                        "server.port=9090\n"
+        );
+        editor.assertProblems(
+                "spring.application.name|Duplicate",
+                "spring.application.name|Duplicate",
+                "server.port|Duplicate",
+                "server.port|Duplicate"
+        );
+
+        //nitpick 1: leading spaces before the marker means... it is not a marker
+        editor = newEditor(
+                "spring.config.activate.on-profile=foo\n" +
+                        "server.port=8888\n" +
+                        " !---\n" +
+                        "spring.config.activate.on-profile=bar\n" +
+                        "server.port=8080\n"
+        );
+        editor.assertProblems(
+                "spring.config.activate.on-profile|Duplicate",
+                "server.port|Duplicate",
+                "spring.config.activate.on-profile|Duplicate",
+                "server.port|Duplicate"
+        );
+
+        //nitpick 2: trailing spaces after the marker are ignored
+        editor = newEditor(
+                "spring.config.activate.on-profile=foo\n" +
+                        "server.port=8888\n" +
+                        "!---   \t\n" +
+                        "spring.config.activate.on-profile=bar\n" +
+                        "server.port=8080\n"
+        );
+        editor.assertProblems(/*NONE*/);
+        
+        editor = newEditor(
+                "spring.application.name=frodo\n" +
+                        "spring.config.activate.on-profile=foo\n" +
+                        "server.port=8888\n" +
+                        "!---\n" +
+                        "! Something off\n" +
+                        "spring.application.name=sam\n" +
+                        "spring.config.activate.on-profile=bar\n" +
+                        "server.port=9090\n"
+        );
+        editor.assertProblems(
+                "spring.application.name|Duplicate",
+                "spring.config.activate.on-profile|Duplicate",
+                "server.port|Duplicate",
+                "spring.application.name|Duplicate",
+                "spring.config.activate.on-profile|Duplicate",
+                "server.port|Duplicate"
+        );
+
+        editor = newEditor(
+                "spring.application.name=frodo\n" +
+                        "spring.config.activate.on-profile=foo\n" +
+                        "server.port=8888\n" +
+                        "!---\n" +
+                        "# Something off\n" +
+                        "spring.application.name=sam\n" +
+                        "spring.config.activate.on-profile=bar\n" +
+                        "server.port=9090\n"
+        );
+        editor.assertProblems(/*NONE*/);
     }
     
     @Test