modify something

pull/451/head
maxisvest 3 years ago
parent 7be9fd542a
commit bea61eadb3

@ -0,0 +1,78 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>cn.hippo4j</groupId>
<artifactId>hippo4j-example</artifactId>
<version>${revision}</version>
</parent>
<artifactId>hippo4j-spring-boot-starter-es-monitor-example</artifactId>
<properties>
<maven.deploy.skip>true</maven.deploy.skip>
</properties>
<dependencies>
<dependency>
<groupId>cn.hippo4j</groupId>
<artifactId>hippo4j-example-core</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>cn.hippo4j</groupId>
<artifactId>hippo4j-core-spring-boot-starter</artifactId>
<version>${revision}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>${tomcat-embed-core.version}</version>
</dependency>
<!--<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>-->
<!--<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-undertow</artifactId>
</dependency>-->
<!--<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jetty</artifactId>
</dependency>-->
</dependencies>
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.hippo4j.example.es.monitor;
import cn.hippo4j.core.enable.EnableDynamicThreadPool;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@EnableDynamicThreadPool
@SpringBootApplication(scanBasePackages = {"cn.hippo4j.example.core", "cn.hippo4j.monitor"})
public class Hippo4JExampleEsMonitorApplication {
public static void main(String[] args) {
SpringApplication.run(Hippo4JExampleEsMonitorApplication.class, args);
}
}

@ -0,0 +1,41 @@
server.port=8088
server.servlet.context-path=/example
spring.profiles.active=dev
spring.application.name=dynamic-threadpool-example
es.host = xxx
es.scheme = xxx
es.userName = xxx
es.password = xxx
es.thread-pool-state.index.name = thread-pool-state
spring.dynamic.thread-pool.item-id=test
spring.dynamic.thread-pool.enable=true
spring.dynamic.thread-pool.banner=false
spring.dynamic.thread-pool.collect=true
spring.dynamic.thread-pool.collect-type=es
spring.dynamic.thread-pool.notify-platforms[0].platform=DING
spring.dynamic.thread-pool.notify-platforms[0].token=xxx
spring.dynamic.thread-pool.notify-platforms[0].secret=xxx
spring.dynamic.thread-pool.apollo.namespace=threadpool.yaml
spring.dynamic.thread-pool.config-file-type=yaml
spring.dynamic.thread-pool.alarm=true
spring.dynamic.thread-pool.check-state-interval=3000
spring.dynamic.thread-pool.capacity-alarm=80
spring.dynamic.thread-pool.alarm-interval=8
spring.dynamic.thread-pool.executors[0].thread-pool-id=message-consume
spring.dynamic.thread-pool.executors[0].core-pool-size=32
spring.dynamic.thread-pool.executors[0].maximum-pool-size=32
spring.dynamic.thread-pool.executors[0].queue-capacity=999
spring.dynamic.thread-pool.executors[0].execute-time-out=1500
spring.dynamic.thread-pool.executors[0].blocking-queue=ResizableCapacityLinkedBlockIngQueue
spring.dynamic.thread-pool.executors[0].rejected-handler=CallerRunsPolicy
spring.dynamic.thread-pool.executors[0].keep-alive-time=1024
spring.dynamic.thread-pool.executors[0].allow-core-thread-time-out=true
spring.dynamic.thread-pool.executors[0].thread-name-prefix=untimely-thread-pool
spring.dynamic.thread-pool.executors[0].notify.is-alarm=true
spring.dynamic.thread-pool.executors[0].notify.capacity-alarm=20
spring.dynamic.thread-pool.executors[0].notify.interval=2
spring.dynamic.thread-pool.executors[0].notify.receive=xxx

@ -0,0 +1,81 @@
package cn.hippo4j.monitor.es;
import cn.hippo4j.common.config.ApplicationContextHolder;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.Strings;
import org.springframework.core.env.Environment;
import java.util.List;
/**
* Create by yuyang
* 2022/8/4 16:26
*/
@Slf4j
public class EsClientHolder {
private static String host;
private static String scheme;
private static String userName;
private static String password;
private static RestHighLevelClient client;
private static RestHighLevelClient initRestClient() {
try {
Environment environment = ApplicationContextHolder.getInstance().getEnvironment();
host = environment.getProperty("es.host");
scheme = environment.getProperty("es.schema");
userName = environment.getProperty("es.userName");
password = environment.getProperty("es.password");
List<HttpHost> hosts = parseHosts();
if (Strings.isNullOrEmpty(userName) || Strings.isNullOrEmpty(password)) {
client = new RestHighLevelClient(RestClient.builder(hosts.toArray(new HttpHost[]{})));
} else {
client = new RestHighLevelClient(RestClient.builder(hosts.toArray(new HttpHost[]{}))
.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setDefaultCredentialsProvider(getCredentialsProvider())));
}
log.info("[ES RestHighLevelClient] success to connect eshost:{},scheme:{}", host, scheme);
return client;
} catch (Exception ex) {
log.error("[ES RestHighLevelClient] fail to connect es! cause:{}", Throwables.getStackTraceAsString(ex));
}
return null;
}
private static BasicCredentialsProvider getCredentialsProvider() {
if (!Strings.isNullOrEmpty(userName) && !Strings.isNullOrEmpty(password)) {
final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(userName, password));
return credentialsProvider;
}
return null;
}
public static RestHighLevelClient getClient() {
return null == client ? initRestClient() : client;
}
private static List<HttpHost> parseHosts() {
String[] hostAndPorts = host.split(",");
List<HttpHost> hosts = Lists.newArrayList();
for (String hostAndPort : hostAndPorts) {
hosts.add(new HttpHost(hostAndPort.split(":")[0], Integer.parseInt(hostAndPort.split(":")[1]), scheme));
}
return hosts;
}
}

@ -19,54 +19,128 @@ package cn.hippo4j.monitor.es;
import cn.hippo4j.common.config.ApplicationContextHolder;
import cn.hippo4j.common.model.ThreadPoolRunStateInfo;
import cn.hippo4j.common.toolkit.JSONUtil;
import cn.hippo4j.core.executor.state.ThreadPoolRunStateHandler;
import cn.hippo4j.monitor.es.model.EsThreadPoolRunStateInfo;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.io.FileUtil;
import com.example.monitor.base.AbstractDynamicThreadPoolMonitor;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.core.env.Environment;
import org.springframework.util.StringUtils;
import java.util.Map;
/**
* Prometheus monitor handler.
*/
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
@Slf4j
public class EsMonitorHandler extends AbstractDynamicThreadPoolMonitor {
private final Map<String, ThreadPoolRunStateInfo> RUN_STATE_CACHE = Maps.newConcurrentMap();
public EsMonitorHandler(ThreadPoolRunStateHandler threadPoolRunStateHandler) {
super(threadPoolRunStateHandler);
}
private AtomicBoolean isIndexExist = null;
@Override
protected void execute(ThreadPoolRunStateInfo poolRunStateInfo) {
Environment environment = ApplicationContextHolder.getInstance().getEnvironment();
RestHighLevelClient esClient = ApplicationContextHolder.getBean(RestHighLevelClient.class);
EsThreadPoolRunStateInfo esThreadPoolRunStateInfo = new EsThreadPoolRunStateInfo();
BeanUtil.copyProperties(poolRunStateInfo, esThreadPoolRunStateInfo);
Environment environment = ApplicationContextHolder.getInstance().getEnvironment();
String indexName = environment.getProperty("es.index.name", "thread-pool-state");
String applicationName = environment.getProperty("spring.application.name", "application");
// Iterable<Tag> tags = Lists.newArrayList(
// Tag.of(DYNAMIC_THREAD_POOL_ID_TAG, poolRunStateInfo.getTpId()),
// Tag.of(APPLICATION_NAME_TAG, applicationName));
// // load
// Metrics.gauge(metricName("current.load"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getSimpleCurrentLoad);
// Metrics.gauge(metricName("peak.load"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getSimplePeakLoad);
// // thread pool
// Metrics.gauge(metricName("core.size"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getCoreSize);
// Metrics.gauge(metricName("maximum.size"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getMaximumSize);
// Metrics.gauge(metricName("current.size"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getPoolSize);
// Metrics.gauge(metricName("largest.size"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getLargestPoolSize);
// Metrics.gauge(metricName("active.size"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getActiveSize);
// // queue
// Metrics.gauge(metricName("queue.size"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getQueueSize);
// Metrics.gauge(metricName("queue.capacity"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getQueueCapacity);
// Metrics.gauge(metricName("queue.remaining.capacity"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getQueueRemainingCapacity);
// // other
// Metrics.gauge(metricName("completed.task.count"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getCompletedTaskCount);
// Metrics.gauge(metricName("reject.count"), tags, poolRunStateInfo, ThreadPoolRunStateInfo::getRejectCount);
if (!this.isExists(indexName)) {
List<String> rawMapping = FileUtil.readLines(new File(Thread.currentThread().getContextClassLoader().getResource("mapping.json").getPath()), StandardCharsets.UTF_8);
String mapping = String.join(" ", rawMapping);
//if index doesn't exsit, this function may try to create one, but recommend to create index manually.
this.createIndex(indexName, "_doc", mapping, null, null, null);
}
esThreadPoolRunStateInfo.setApplicationName(applicationName);
esThreadPoolRunStateInfo.setId("thread-pool-state-" + System.currentTimeMillis());
this.log2Es(esThreadPoolRunStateInfo, indexName);
}
public void log2Es(EsThreadPoolRunStateInfo esThreadPoolRunStateInfo, String indexName) {
RestHighLevelClient client = EsClientHolder.getClient();
try {
IndexRequest request = new IndexRequest(indexName, "_doc");
request.id(esThreadPoolRunStateInfo.getId());
String stateJson = JSONUtil.toJSONString(esThreadPoolRunStateInfo);
request.source(stateJson, XContentType.JSON);
IndexResponse response = client.index(request, RequestOptions.DEFAULT);
log.info("write thread-pool state to es:{}", stateJson);
} catch (Exception ex) {
log.error("es index error, the exception was thrown in create index. name:{},type:{},id:{}. {} ",
indexName,
"_doc",
esThreadPoolRunStateInfo.getId(),
ex);
}
}
public synchronized boolean isExists(String index) {
if (Objects.isNull(isIndexExist)) {
boolean exists = false;
GetIndexRequest request = new GetIndexRequest();
request.indices(index);
try {
RestHighLevelClient client = EsClientHolder.getClient();
exists = client.indices().exists(request, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
isIndexExist = new AtomicBoolean(exists);
}
return isIndexExist.get();
}
public boolean createIndex(String index, String type, String mapping, Integer shards, Integer replicas, String alias) {
RestHighLevelClient client = EsClientHolder.getClient();
boolean acknowledged = false;
CreateIndexRequest request = new CreateIndexRequest(index);
if (StringUtils.hasText(mapping)) {
request.mapping(type, mapping, XContentType.JSON);
}
if (!Objects.isNull(shards) && !Objects.isNull(replicas)) {
request.settings(Settings.builder()
.put("index.number_of_shards", shards) //5
.put("index.number_of_replicas", replicas));//1
}
if (StringUtils.hasText(alias)) {
request.alias(new Alias(alias));
}
try {
CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
acknowledged = createIndexResponse.isAcknowledged();
} catch (IOException e) {
e.printStackTrace();
}
if (acknowledged) {
log.info("create es index success");
isIndexExist.set(true);
} else {
log.error("create es index fail");
throw new RuntimeException("cannot auto create thread-pool state es index");
}
return acknowledged;
}
@Override

@ -1,148 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cn.hippo4j.monitor.es.config;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.List;
@Slf4j
@Configuration
@ConfigurationProperties(prefix = "es")
@Getter
@Setter
public class EsConfig {
// cluster name
private String cluster;
// such as ip1:9200,ip2:9200
private String host;
// such as :http
private String scheme;
private String userName;
private String password;
private int timeout;
private RestHighLevelClient client;
@Getter
private SearchSourceBuilder searchBuilder;
public EsConfig() {
}
public EsConfig(String host) {
initRestClient(host, null, null);
}
public EsConfig(String host, String userName, String password) {
initRestClient(host, userName, password);
}
public RestHighLevelClient initRestClient(String host, String userName, String password) {
this.host = host;
this.userName = userName;
this.password = password;
return initRestClient();
}
@Bean(destroyMethod = "close")
protected RestHighLevelClient initRestClient() {
try {
List<HttpHost> hosts = parseHosts();
if (Strings.isNullOrEmpty(userName) || Strings.isNullOrEmpty(password)) {
client = new RestHighLevelClient(RestClient.builder(hosts.toArray(new HttpHost[]{})));
} else {
client = new RestHighLevelClient(RestClient.builder(hosts.toArray(new HttpHost[]{}))
.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder.setDefaultCredentialsProvider(getCredentialsProvider())));
}
log.info("[ES RestHighLevelClient] ES连接成功host:{},scheme:{}", host, scheme);
return client;
} catch (Exception ex) {
log.error("[ES RestHighLevelClient] ES连接失败! cause:{}", Throwables.getStackTraceAsString(ex));
}
return null;
}
@Bean
protected BasicCredentialsProvider getCredentialsProvider() {
if (!Strings.isNullOrEmpty(userName) && !Strings.isNullOrEmpty(password)) {
final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(userName, password));
return credentialsProvider;
}
return null;
}
public RestHighLevelClient getClient() {
return null == client ? initRestClient() : client;
}
@Bean
public SearchSourceBuilder getSearchSourceBuilder() {
if (null == searchBuilder) {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.timeout(new TimeValue(timeout));
searchBuilder = sourceBuilder;
}
return searchBuilder;
}
private List<HttpHost> parseHosts() {
String[] hostAndPorts = host.split(",");
List<HttpHost> hosts = Lists.newArrayList();
for (String hostAndPort : hostAndPorts) {
hosts.add(new HttpHost(hostAndPort.split(":")[0], Integer.parseInt(hostAndPort.split(":")[1]), scheme));
}
return hosts;
}
public void printDebugConfigs() {
String debug = "EsSettings: \n" +
"cluster:" + cluster + "\n" +
"host:" + host + "\n" +
"scheme:" + scheme + "\n" +
"userName:" + userName + "\n" +
"password:" + password + "\n" +
"timeout:" + timeout;
log.info(debug);
}
}

@ -0,0 +1,19 @@
package cn.hippo4j.monitor.es.model;
import cn.hippo4j.common.model.ThreadPoolRunStateInfo;
import lombok.Getter;
import lombok.Setter;
/**
* Create by yuyang
* 2022/8/4 17:17
*/
@Getter
@Setter
public class EsThreadPoolRunStateInfo extends ThreadPoolRunStateInfo{
private String Id;
private String applicationName;
}

@ -0,0 +1,78 @@
{
"_doc": {
"properties": {
"activeSize": {
"type": "long"
},
"applicationName": {
"type": "keyword"
},
"clientLastRefreshTime": {
"type": "text"
},
"completedTaskCount": {
"type": "long"
},
"coreSize": {
"type": "long"
},
"currentLoad": {
"type": "text"
},
"freeMemory": {
"type": "text"
},
"host": {
"type": "keyword"
},
"id": {
"type": "text"
},
"largestPoolSize": {
"type": "long"
},
"maximumSize": {
"type": "long"
},
"memoryProportion": {
"type": "text"
},
"peakLoad": {
"type": "text"
},
"poolSize": {
"type": "long"
},
"queueCapacity": {
"type": "long"
},
"queueRemainingCapacity": {
"type": "long"
},
"queueSize": {
"type": "long"
},
"queueType": {
"type": "text"
},
"rejectCount": {
"type": "long"
},
"rejectedName": {
"type": "text"
},
"simpleCurrentLoad": {
"type": "long"
},
"simplePeakLoad": {
"type": "long"
},
"timestamp": {
"type": "date"
},
"tpId": {
"type": "keyword"
}
}
}
}

@ -59,7 +59,8 @@ import org.springframework.core.annotation.Order;
@Import({
ConfigHandlerConfiguration.EmbeddedNacos.class, ConfigHandlerConfiguration.EmbeddedNacosCloud.class,
ConfigHandlerConfiguration.EmbeddedApollo.class, ConfigHandlerConfiguration.EmbeddedZookeeper.class,
MonitorHandlerConfiguration.EmbeddedLogMonitor.class, MonitorHandlerConfiguration.EmbeddedPrometheusMonitor.class
MonitorHandlerConfiguration.EmbeddedLogMonitor.class, MonitorHandlerConfiguration.EmbeddedPrometheusMonitor.class,
MonitorHandlerConfiguration.EmbeddedEsMonitor.class
})
public class DynamicThreadPoolCoreAutoConfiguration {

@ -31,7 +31,7 @@ public class EsMonitorCondition implements Condition {
@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
String collectType = context.getEnvironment().getProperty(BootstrapCoreProperties.PREFIX + "collect-type", "");
String collectType = context.getEnvironment().getProperty(BootstrapCoreProperties.PREFIX + ".collect-type", "");
return StringUtil.isNotEmpty(collectType) && collectType.contains(MonitorTypeEnum.ES.name().toLowerCase()) ? true : false;
}
}

Loading…
Cancel
Save