diff --git a/config/src/main/java/org/springframework/security/config/http/AuthorizationFilterParser.java b/config/src/main/java/org/springframework/security/config/http/AuthorizationFilterParser.java
index e3ac6c915a..1dcd2888c0 100644
--- a/config/src/main/java/org/springframework/security/config/http/AuthorizationFilterParser.java
+++ b/config/src/main/java/org/springframework/security/config/http/AuthorizationFilterParser.java
@@ -50,6 +50,8 @@ class AuthorizationFilterParser implements BeanDefinitionParser {
 
 	private static final String ATT_USE_EXPRESSIONS = "use-expressions";
 
+	private static final String ATT_ACCESS_DECISION_MANAGER_REF = "access-decision-manager-ref";
+
 	private static final String ATT_HTTP_METHOD = "method";
 
 	private static final String ATT_PATTERN = "pattern";
@@ -60,6 +62,12 @@ class AuthorizationFilterParser implements BeanDefinitionParser {
 
 	private String authorizationManagerRef;
 
+	private final BeanMetadataElement securityContextHolderStrategy;
+
+	AuthorizationFilterParser(BeanMetadataElement securityContextHolderStrategy) {
+		this.securityContextHolderStrategy = securityContextHolderStrategy;
+	}
+
 	@Override
 	public BeanDefinition parse(Element element, ParserContext parserContext) {
 		if (!isUseExpressions(element)) {
@@ -67,10 +75,16 @@ class AuthorizationFilterParser implements BeanDefinitionParser {
 					element);
 			return null;
 		}
+		if (StringUtils.hasText(element.getAttribute(ATT_ACCESS_DECISION_MANAGER_REF))) {
+			parserContext.getReaderContext().error(
+					"AuthorizationManager cannot be used in conjunction with `access-decision-manager-ref`", element);
+			return null;
+		}
 		this.authorizationManagerRef = createAuthorizationManager(element, parserContext);
 		BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.rootBeanDefinition(AuthorizationFilter.class);
 		filterBuilder.getRawBeanDefinition().setSource(parserContext.extractSource(element));
 		BeanDefinition filter = filterBuilder.addConstructorArgReference(this.authorizationManagerRef)
+				.addPropertyValue("securityContextHolderStrategy", this.securityContextHolderStrategy)
 				.getBeanDefinition();
 		String id = element.getAttribute(AbstractBeanDefinitionParser.ID_ATTRIBUTE);
 		if (StringUtils.hasText(id)) {
@@ -172,7 +186,9 @@ class AuthorizationFilterParser implements BeanDefinitionParser {
 
 		@Override
 		public DefaultHttpSecurityExpressionHandler getBean() {
-			this.handler.setDefaultRolePrefix(this.rolePrefix);
+			if (this.rolePrefix != null) {
+				this.handler.setDefaultRolePrefix(this.rolePrefix);
+			}
 			return this.handler;
 		}
 
diff --git a/config/src/main/java/org/springframework/security/config/http/HttpConfigurationBuilder.java b/config/src/main/java/org/springframework/security/config/http/HttpConfigurationBuilder.java
index 9bc23955bb..32c2fd9d73 100644
--- a/config/src/main/java/org/springframework/security/config/http/HttpConfigurationBuilder.java
+++ b/config/src/main/java/org/springframework/security/config/http/HttpConfigurationBuilder.java
@@ -729,7 +729,7 @@ class HttpConfigurationBuilder {
 	}
 
 	private void createAuthorizationFilter() {
-		AuthorizationFilterParser authorizationFilterParser = new AuthorizationFilterParser();
+		AuthorizationFilterParser authorizationFilterParser = new AuthorizationFilterParser(this.holderStrategyRef);
 		BeanDefinition fsiBean = authorizationFilterParser.parse(this.httpElt, this.pc);
 		String fsiId = this.pc.getReaderContext().generateBeanName(fsiBean);
 		this.pc.registerBeanComponent(new BeanComponentDefinition(fsiBean, fsiId));