feat:support fault injection. (#1691)
Co-authored-by: Haotian Zhang <skyebefreeman@qq.com>pull/1692/head
Fishtail
1 month ago
committed by
GitHub
parent
e86f7c7f1c
commit
fdec06660a
@ -0,0 +1,29 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>spring-cloud-tencent-plugin-starters</artifactId>
|
||||
<groupId>com.tencent.cloud</groupId>
|
||||
<version>${revision}</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>spring-cloud-starter-tencent-fault-injection-plugin</artifactId>
|
||||
<name>Spring Cloud Starter Tencent Fault Injection Plugin</name>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.tencent.cloud</groupId>
|
||||
<artifactId>spring-cloud-tencent-rpc-enhancement</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,118 @@
|
||||
/*
|
||||
* Tencent is pleased to support the open source community by making spring-cloud-tencent available.
|
||||
*
|
||||
* Copyright (C) 2021 Tencent. All rights reserved.
|
||||
*
|
||||
* Licensed under the BSD 3-Clause License (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://opensource.org/licenses/BSD-3-Clause
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed
|
||||
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.tencent.cloud.plugin.fault;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
import com.tencent.cloud.common.constant.ContextConstant;
|
||||
import com.tencent.cloud.common.metadata.MetadataContext;
|
||||
import com.tencent.cloud.common.metadata.MetadataContextHolder;
|
||||
import com.tencent.cloud.plugin.fault.config.FaultInjectionProperties;
|
||||
import com.tencent.cloud.plugin.fault.instrument.resttemplate.PolarisFaultInjectionHttpResponse;
|
||||
import com.tencent.cloud.rpc.enhancement.plugin.EnhancedPlugin;
|
||||
import com.tencent.cloud.rpc.enhancement.plugin.EnhancedPluginContext;
|
||||
import com.tencent.cloud.rpc.enhancement.plugin.EnhancedPluginType;
|
||||
import com.tencent.cloud.rpc.enhancement.plugin.EnhancedRequestContext;
|
||||
import com.tencent.polaris.api.pojo.CircuitBreakerStatus;
|
||||
import com.tencent.polaris.api.utils.ClassUtils;
|
||||
import com.tencent.polaris.api.utils.StringUtils;
|
||||
import com.tencent.polaris.fault.api.core.FaultAPI;
|
||||
import com.tencent.polaris.fault.api.rpc.AbortResult;
|
||||
import com.tencent.polaris.fault.api.rpc.DelayResult;
|
||||
import com.tencent.polaris.fault.api.rpc.FaultRequest;
|
||||
import com.tencent.polaris.fault.api.rpc.FaultResponse;
|
||||
import com.tencent.polaris.fault.client.exception.FaultInjectionException;
|
||||
import com.tencent.polaris.metadata.core.MetadataType;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static com.tencent.cloud.common.metadata.MetadataContext.LOCAL_NAMESPACE;
|
||||
import static com.tencent.cloud.common.metadata.MetadataContext.LOCAL_SERVICE;
|
||||
import static com.tencent.cloud.rpc.enhancement.plugin.PluginOrderConstant.ClientPluginOrder.FAULT_INJECTION_PLUGIN_ORDER;
|
||||
|
||||
/**
|
||||
* Fault injection pre plugin.
|
||||
*
|
||||
* @author Haotian Zhang
|
||||
*/
|
||||
public class FaultInjectionPrePlugin implements EnhancedPlugin {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(FaultInjectionPrePlugin.class);
|
||||
|
||||
private final FaultAPI faultAPI;
|
||||
|
||||
private final FaultInjectionProperties faultInjectionProperties;
|
||||
|
||||
public FaultInjectionPrePlugin(FaultAPI faultAPI, FaultInjectionProperties faultInjectionProperties) {
|
||||
this.faultAPI = faultAPI;
|
||||
this.faultInjectionProperties = faultInjectionProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return FaultInjectionPrePlugin.class.getName();
|
||||
}
|
||||
|
||||
@Override
|
||||
public EnhancedPluginType getType() {
|
||||
return EnhancedPluginType.Client.PRE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(EnhancedPluginContext context) throws Throwable {
|
||||
if (!faultInjectionProperties.isEnabled()) {
|
||||
return;
|
||||
}
|
||||
|
||||
EnhancedRequestContext request = context.getRequest();
|
||||
String governanceNamespace = StringUtils.isNotEmpty(request.getGovernanceNamespace()) ? request.getGovernanceNamespace() : MetadataContext.LOCAL_NAMESPACE;
|
||||
FaultRequest faultRequest = new FaultRequest(LOCAL_NAMESPACE, LOCAL_SERVICE, governanceNamespace, request.getHost(), MetadataContextHolder.get());
|
||||
|
||||
FaultResponse faultResponse = faultAPI.fault(faultRequest);
|
||||
if (faultResponse.isFaultInjected()) {
|
||||
if (faultResponse.getDelayResult() != null && faultResponse.getDelayResult().getDelay() > 0) {
|
||||
DelayResult delayResult = faultResponse.getDelayResult();
|
||||
Thread.sleep(delayResult.getDelay());
|
||||
}
|
||||
if (faultResponse.getAbortResult() != null) {
|
||||
AbortResult abortResult = faultResponse.getAbortResult();
|
||||
CircuitBreakerStatus.FallbackInfo fallbackInfo = new CircuitBreakerStatus.FallbackInfo(abortResult.getAbortCode(), new HashMap<>(), "");
|
||||
if (ClassUtils.isClassPresent("org.springframework.http.client.ClientHttpResponse")) {
|
||||
Object fallbackResponse = new PolarisFaultInjectionHttpResponse(fallbackInfo);
|
||||
putMetadataObjectValue(ContextConstant.FaultInjection.FAULT_INJECTION_FALLBACK_HTTP_RESPONSE, fallbackResponse);
|
||||
}
|
||||
throw new FaultInjectionException(fallbackInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void putMetadataObjectValue(String key, Object value) {
|
||||
MetadataContextHolder.get().getMetadataContainer(MetadataType.APPLICATION, true).
|
||||
putMetadataObjectValue(key, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handlerThrowable(EnhancedPluginContext context, Throwable throwable) {
|
||||
LOG.error("FaultInjectionPrePlugin runs failed. context=[{}].", context, throwable);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getOrder() {
|
||||
return FAULT_INJECTION_PLUGIN_ORDER;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Tencent is pleased to support the open source community by making spring-cloud-tencent available.
|
||||
*
|
||||
* Copyright (C) 2021 Tencent. All rights reserved.
|
||||
*
|
||||
* Licensed under the BSD 3-Clause License (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://opensource.org/licenses/BSD-3-Clause
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed
|
||||
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.tencent.cloud.plugin.fault.config;
|
||||
|
||||
import com.tencent.cloud.plugin.fault.FaultInjectionPrePlugin;
|
||||
import com.tencent.cloud.polaris.context.PolarisSDKContextManager;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/**
|
||||
* Autoconfiguration for fault injection plugin.
|
||||
*
|
||||
* @author Haotian Zhang
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@ConditionalOnProperty(value = "spring.cloud.polaris.fault-injection.enabled", matchIfMissing = true)
|
||||
@EnableConfigurationProperties(FaultInjectionProperties.class)
|
||||
public class FaultInjectionAutoConfiguration {
|
||||
|
||||
@Bean
|
||||
public FaultInjectionModifier faultInjectionModifier(FaultInjectionProperties faultInjectionProperties) {
|
||||
return new FaultInjectionModifier(faultInjectionProperties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public FaultInjectionPrePlugin faultInjectionPrePlugin(PolarisSDKContextManager polarisSDKContextManager,
|
||||
FaultInjectionProperties faultInjectionProperties) {
|
||||
return new FaultInjectionPrePlugin(polarisSDKContextManager.getFaultAPI(), faultInjectionProperties);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Tencent is pleased to support the open source community by making spring-cloud-tencent available.
|
||||
*
|
||||
* Copyright (C) 2021 Tencent. All rights reserved.
|
||||
*
|
||||
* Licensed under the BSD 3-Clause License (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://opensource.org/licenses/BSD-3-Clause
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed
|
||||
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.tencent.cloud.plugin.fault.config;
|
||||
|
||||
import com.tencent.cloud.common.constant.OrderConstant;
|
||||
import com.tencent.cloud.polaris.context.PolarisConfigModifier;
|
||||
import com.tencent.polaris.factory.config.ConfigurationImpl;
|
||||
import com.tencent.polaris.factory.config.consumer.FaultConfigImpl;
|
||||
|
||||
/**
|
||||
* Properties for fault injection.
|
||||
*
|
||||
* @author Haotian Zhang
|
||||
*/
|
||||
public class FaultInjectionModifier implements PolarisConfigModifier {
|
||||
|
||||
private final FaultInjectionProperties faultInjectionProperties;
|
||||
|
||||
public FaultInjectionModifier(FaultInjectionProperties faultInjectionProperties) {
|
||||
this.faultInjectionProperties = faultInjectionProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void modify(ConfigurationImpl configuration) {
|
||||
FaultConfigImpl faultConfig = (FaultConfigImpl) configuration.getConsumer().getFault();
|
||||
if (faultInjectionProperties.isEnabled()) {
|
||||
faultInjectionProperties.setEnabled(true);
|
||||
}
|
||||
else {
|
||||
faultConfig.setEnable(false);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getOrder() {
|
||||
return OrderConstant.Modifier.FAULT_INJECTION_ORDER;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Tencent is pleased to support the open source community by making spring-cloud-tencent available.
|
||||
*
|
||||
* Copyright (C) 2021 Tencent. All rights reserved.
|
||||
*
|
||||
* Licensed under the BSD 3-Clause License (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://opensource.org/licenses/BSD-3-Clause
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed
|
||||
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.tencent.cloud.plugin.fault.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
/**
|
||||
* Properties for fault injection.
|
||||
*
|
||||
* @author Haotian Zhang
|
||||
*/
|
||||
@ConfigurationProperties("spring.cloud.polaris.fault-injection")
|
||||
public class FaultInjectionProperties {
|
||||
|
||||
/**
|
||||
* If traffic mirroring is enabled. Default is true.
|
||||
*/
|
||||
private boolean enabled = true;
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "FaultInjectionProperties{" +
|
||||
"enabled=" + enabled +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,106 @@
|
||||
/*
|
||||
* Tencent is pleased to support the open source community by making spring-cloud-tencent available.
|
||||
*
|
||||
* Copyright (C) 2021 Tencent. All rights reserved.
|
||||
*
|
||||
* Licensed under the BSD 3-Clause License (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://opensource.org/licenses/BSD-3-Clause
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed
|
||||
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.tencent.cloud.plugin.fault.instrument.resttemplate;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
|
||||
import com.tencent.polaris.api.pojo.CircuitBreakerStatus;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.HttpStatusCode;
|
||||
import org.springframework.http.client.ClientHttpResponse;
|
||||
|
||||
/**
|
||||
* PolarisFaultInjectionHttpResponse.
|
||||
*
|
||||
* @author Haotian Zhang
|
||||
*/
|
||||
public class PolarisFaultInjectionHttpResponse implements ClientHttpResponse {
|
||||
|
||||
private final CircuitBreakerStatus.FallbackInfo fallbackInfo;
|
||||
|
||||
private HttpHeaders headers = new HttpHeaders();
|
||||
|
||||
private InputStream body;
|
||||
|
||||
public PolarisFaultInjectionHttpResponse(int code) {
|
||||
this(new CircuitBreakerStatus.FallbackInfo(code, null, null));
|
||||
}
|
||||
|
||||
public PolarisFaultInjectionHttpResponse(int code, String body) {
|
||||
this(new CircuitBreakerStatus.FallbackInfo(code, null, body));
|
||||
}
|
||||
|
||||
public PolarisFaultInjectionHttpResponse(int code, Map<String, String> headers, String body) {
|
||||
this(new CircuitBreakerStatus.FallbackInfo(code, headers, body));
|
||||
}
|
||||
|
||||
public PolarisFaultInjectionHttpResponse(CircuitBreakerStatus.FallbackInfo fallbackInfo) {
|
||||
this.fallbackInfo = fallbackInfo;
|
||||
if (fallbackInfo.getHeaders() != null) {
|
||||
fallbackInfo.getHeaders().forEach(headers::add);
|
||||
}
|
||||
if (fallbackInfo.getBody() != null) {
|
||||
body = new ByteArrayInputStream(fallbackInfo.getBody().getBytes());
|
||||
}
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public HttpStatusCode getStatusCode() {
|
||||
return HttpStatus.valueOf(fallbackInfo.getCode());
|
||||
}
|
||||
|
||||
@NotNull
|
||||
@Override
|
||||
public final String getStatusText() {
|
||||
HttpStatus status = HttpStatus.resolve(getStatusCode().value());
|
||||
return (status != null ? status.getReasonPhrase() : "");
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void close() {
|
||||
if (this.body != null) {
|
||||
try {
|
||||
this.body.close();
|
||||
}
|
||||
catch (IOException e) {
|
||||
// Ignore exception on close...
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public final InputStream getBody() {
|
||||
return this.body;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final HttpHeaders getHeaders() {
|
||||
return this.headers;
|
||||
}
|
||||
|
||||
public CircuitBreakerStatus.FallbackInfo getFallbackInfo() {
|
||||
return this.fallbackInfo;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1 @@
|
||||
com.tencent.cloud.plugin.fault.config.FaultInjectionAutoConfiguration
|
||||
@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Tencent is pleased to support the open source community by making spring-cloud-tencent available.
|
||||
*
|
||||
* Copyright (C) 2021 Tencent. All rights reserved.
|
||||
*
|
||||
* Licensed under the BSD 3-Clause License (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://opensource.org/licenses/BSD-3-Clause
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed
|
||||
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
||||
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
|
||||
package com.tencent.cloud.rpc.enhancement.config;
|
||||
|
||||
import com.tencent.cloud.polaris.context.ConditionalOnPolarisEnabled;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
/**
|
||||
* Auto Configuration for Polaris {@link feign.Feign} OR {@link RestTemplate} which can automatically bring in the call
|
||||
* results for reporting.
|
||||
*
|
||||
* @author <a href="mailto:iskp.me@gmail.com">Palmer.Xu</a> 2022-06-29
|
||||
*/
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@ConditionalOnPolarisEnabled
|
||||
public class RpcEnhancementPropertiesAutoConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public RpcEnhancementReporterProperties rpcEnhancementReporterProperties() {
|
||||
return new RpcEnhancementReporterProperties();
|
||||
}
|
||||
}
|
||||
@ -1,3 +1,3 @@
|
||||
org.springframework.cloud.bootstrap.BootstrapConfiguration=\
|
||||
com.tencent.cloud.rpc.enhancement.stat.config.PolarisStatPropertiesBootstrapConfiguration,\
|
||||
com.tencent.cloud.rpc.enhancement.config.RpcEnhancementBootstrapConfiguration
|
||||
com.tencent.cloud.rpc.enhancement.config.RpcEnhancementPropertiesBootstrapConfiguration
|
||||
|
||||
@ -1,2 +1,3 @@
|
||||
com.tencent.cloud.rpc.enhancement.config.RpcEnhancementAutoConfiguration
|
||||
com.tencent.cloud.rpc.enhancement.stat.config.PolarisStatPropertiesAutoConfiguration
|
||||
com.tencent.cloud.rpc.enhancement.config.RpcEnhancementPropertiesAutoConfiguration
|
||||
|
||||
Loading…
Reference in new issue