mirror of https://github.com/longtai-cn/hippo4j
Add es monitor support (#451)
* first commit * modify something * modify something * modify log content * format codepull/467/head
maxisvest
2 years ago
committed by
GitHub
parent
8e7e62c2f5
commit
b34c84f97d
@ -0,0 +1,78 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>cn.hippo4j</groupId>
|
||||
<artifactId>hippo4j-example</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
<artifactId>hippo4j-spring-boot-starter-es-monitor-example</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.deploy.skip>true</maven.deploy.skip>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>cn.hippo4j</groupId>
|
||||
<artifactId>hippo4j-example-core</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>cn.hippo4j</groupId>
|
||||
<artifactId>hippo4j-core-spring-boot-starter</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.tomcat.embed</groupId>
|
||||
<artifactId>tomcat-embed-core</artifactId>
|
||||
<version>${tomcat-embed-core.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!--<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-tomcat</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>-->
|
||||
|
||||
<!--<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-undertow</artifactId>
|
||||
</dependency>-->
|
||||
|
||||
<!--<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-jetty</artifactId>
|
||||
</dependency>-->
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<finalName>${project.artifactId}</finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>repackage</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cn.hippo4j.example.es.monitor;
|
||||
|
||||
import cn.hippo4j.core.enable.EnableDynamicThreadPool;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@EnableDynamicThreadPool
|
||||
@SpringBootApplication(scanBasePackages = {"cn.hippo4j.example.core", "cn.hippo4j.monitor"})
|
||||
public class Hippo4JExampleEsMonitorApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(Hippo4JExampleEsMonitorApplication.class, args);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
server.port=8088
|
||||
|
||||
server.servlet.context-path=/example
|
||||
|
||||
spring.profiles.active=dev
|
||||
spring.application.name=dynamic-threadpool-example
|
||||
|
||||
es.thread-pool-state.host = ip1:port,ip2:port
|
||||
es.thread-pool-state.scheme = http
|
||||
es.thread-pool-state.userName = xxx
|
||||
es.thread-pool-state.password = xxx
|
||||
es.thread-pool-state.index.name = thread-pool-state
|
||||
|
||||
spring.dynamic.thread-pool.item-id=test
|
||||
spring.dynamic.thread-pool.enable=true
|
||||
spring.dynamic.thread-pool.banner=false
|
||||
spring.dynamic.thread-pool.collect=true
|
||||
spring.dynamic.thread-pool.collect-type=es
|
||||
spring.dynamic.thread-pool.notify-platforms[0].platform=DING
|
||||
spring.dynamic.thread-pool.notify-platforms[0].token=xxx
|
||||
spring.dynamic.thread-pool.notify-platforms[0].secret=xxx
|
||||
spring.dynamic.thread-pool.apollo.namespace=threadpool.yaml
|
||||
spring.dynamic.thread-pool.config-file-type=yaml
|
||||
spring.dynamic.thread-pool.alarm=true
|
||||
spring.dynamic.thread-pool.check-state-interval=3000
|
||||
spring.dynamic.thread-pool.capacity-alarm=80
|
||||
spring.dynamic.thread-pool.alarm-interval=8
|
||||
spring.dynamic.thread-pool.executors[0].thread-pool-id=message-consume
|
||||
spring.dynamic.thread-pool.executors[0].core-pool-size=32
|
||||
spring.dynamic.thread-pool.executors[0].maximum-pool-size=32
|
||||
spring.dynamic.thread-pool.executors[0].queue-capacity=999
|
||||
spring.dynamic.thread-pool.executors[0].execute-time-out=1500
|
||||
spring.dynamic.thread-pool.executors[0].blocking-queue=ResizableCapacityLinkedBlockIngQueue
|
||||
spring.dynamic.thread-pool.executors[0].rejected-handler=CallerRunsPolicy
|
||||
spring.dynamic.thread-pool.executors[0].keep-alive-time=1024
|
||||
spring.dynamic.thread-pool.executors[0].allow-core-thread-time-out=true
|
||||
spring.dynamic.thread-pool.executors[0].thread-name-prefix=untimely-thread-pool
|
||||
spring.dynamic.thread-pool.executors[0].notify.is-alarm=true
|
||||
spring.dynamic.thread-pool.executors[0].notify.capacity-alarm=20
|
||||
spring.dynamic.thread-pool.executors[0].notify.interval=2
|
||||
spring.dynamic.thread-pool.executors[0].notify.receive=xxx
|
||||
@ -0,0 +1,81 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>cn.hippo4j</groupId>
|
||||
<artifactId>hippo4j-monitor</artifactId>
|
||||
<version>${revision}</version>
|
||||
</parent>
|
||||
<artifactId>hippo4j-monitor-es</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>cn.hippo4j</groupId>
|
||||
<artifactId>hippo4j-monitor-base</artifactId>
|
||||
<version>${revision}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.micrometer</groupId>
|
||||
<artifactId>micrometer-core</artifactId>
|
||||
<version>${micrometer-core.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch</groupId>
|
||||
<artifactId>elasticsearch</artifactId>
|
||||
<version>${elasticsearch.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>transport</artifactId>
|
||||
<version>${elasticsearch.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>elasticsearch-rest-high-level-client</artifactId>
|
||||
<version>${elasticsearch.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<includes>
|
||||
<include>**/*.txt</include>
|
||||
<include>**/*.json</include>
|
||||
</includes>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<configuration>
|
||||
<archive>
|
||||
<manifestEntries>
|
||||
<Implementation-Title>${project.artifactId}</Implementation-Title>
|
||||
<Implementation-Version>${project.version}</Implementation-Version>
|
||||
<Build-Time>${maven.build.timestamp}</Build-Time>
|
||||
<Built-By>chen.ma</Built-By>
|
||||
</manifestEntries>
|
||||
</archive>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.10.3</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@ -0,0 +1,149 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cn.hippo4j.monitor.es;
|
||||
|
||||
import cn.hippo4j.common.config.ApplicationContextHolder;
|
||||
import cn.hippo4j.common.model.ThreadPoolRunStateInfo;
|
||||
import cn.hippo4j.common.toolkit.JSONUtil;
|
||||
import cn.hippo4j.core.executor.state.ThreadPoolRunStateHandler;
|
||||
import cn.hippo4j.monitor.es.model.EsThreadPoolRunStateInfo;
|
||||
import cn.hutool.core.bean.BeanUtil;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import com.example.monitor.base.AbstractDynamicThreadPoolMonitor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.elasticsearch.action.admin.indices.alias.Alias;
|
||||
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
|
||||
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
|
||||
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.index.IndexResponse;
|
||||
import org.elasticsearch.client.RequestOptions;
|
||||
import org.elasticsearch.client.RestHighLevelClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
@Slf4j
|
||||
public class EsMonitorHandler extends AbstractDynamicThreadPoolMonitor {
|
||||
|
||||
public EsMonitorHandler(ThreadPoolRunStateHandler threadPoolRunStateHandler) {
|
||||
super(threadPoolRunStateHandler);
|
||||
}
|
||||
|
||||
private AtomicBoolean isIndexExist = null;
|
||||
|
||||
@Override
|
||||
protected void execute(ThreadPoolRunStateInfo poolRunStateInfo) {
|
||||
EsThreadPoolRunStateInfo esThreadPoolRunStateInfo = new EsThreadPoolRunStateInfo();
|
||||
BeanUtil.copyProperties(poolRunStateInfo, esThreadPoolRunStateInfo);
|
||||
|
||||
Environment environment = ApplicationContextHolder.getInstance().getEnvironment();
|
||||
String indexName = environment.getProperty("es.thread-pool-state.index.name", "thread-pool-state");
|
||||
String applicationName = environment.getProperty("spring.application.name", "application");
|
||||
|
||||
if (!this.isExists(indexName)) {
|
||||
List<String> rawMapping = FileUtil.readLines(new File(Thread.currentThread().getContextClassLoader().getResource("mapping.json").getPath()), StandardCharsets.UTF_8);
|
||||
String mapping = String.join(" ", rawMapping);
|
||||
// if index doesn't exsit, this function may try to create one, but recommend to create index manually.
|
||||
this.createIndex(indexName, "_doc", mapping, null, null, null);
|
||||
}
|
||||
|
||||
esThreadPoolRunStateInfo.setApplicationName(applicationName);
|
||||
esThreadPoolRunStateInfo.setId(indexName + "-" + System.currentTimeMillis());
|
||||
this.log2Es(esThreadPoolRunStateInfo, indexName);
|
||||
}
|
||||
|
||||
public void log2Es(EsThreadPoolRunStateInfo esThreadPoolRunStateInfo, String indexName) {
|
||||
RestHighLevelClient client = EsClientHolder.getClient();
|
||||
|
||||
try {
|
||||
IndexRequest request = new IndexRequest(indexName, "_doc");
|
||||
request.id(esThreadPoolRunStateInfo.getId());
|
||||
String stateJson = JSONUtil.toJSONString(esThreadPoolRunStateInfo);
|
||||
request.source(stateJson, XContentType.JSON);
|
||||
|
||||
IndexResponse response = client.index(request, RequestOptions.DEFAULT);
|
||||
log.info("write thread-pool state to es, id is :{}", response.getId());
|
||||
} catch (Exception ex) {
|
||||
log.error("es index error, the exception was thrown in create index. name:{},type:{},id:{}. {} ",
|
||||
indexName,
|
||||
"_doc",
|
||||
esThreadPoolRunStateInfo.getId(),
|
||||
ex);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized boolean isExists(String index) {
|
||||
// cache check result
|
||||
if (Objects.isNull(isIndexExist)) {
|
||||
boolean exists = false;
|
||||
GetIndexRequest request = new GetIndexRequest();
|
||||
request.indices(index);
|
||||
try {
|
||||
RestHighLevelClient client = EsClientHolder.getClient();
|
||||
exists = client.indices().exists(request, RequestOptions.DEFAULT);
|
||||
} catch (IOException e) {
|
||||
log.error("check es index fail");
|
||||
}
|
||||
isIndexExist = new AtomicBoolean(exists);
|
||||
}
|
||||
return isIndexExist.get();
|
||||
}
|
||||
|
||||
public void createIndex(String index, String type, String mapping, Integer shards, Integer replicas, String alias) {
|
||||
RestHighLevelClient client = EsClientHolder.getClient();
|
||||
boolean acknowledged = false;
|
||||
CreateIndexRequest request = new CreateIndexRequest(index);
|
||||
if (StringUtils.hasText(mapping)) {
|
||||
request.mapping(type, mapping, XContentType.JSON);
|
||||
}
|
||||
if (!Objects.isNull(shards) && !Objects.isNull(replicas)) {
|
||||
request.settings(Settings.builder()
|
||||
.put("index.number_of_shards", shards) // 5
|
||||
.put("index.number_of_replicas", replicas));// 1
|
||||
}
|
||||
if (StringUtils.hasText(alias)) {
|
||||
request.alias(new Alias(alias));
|
||||
}
|
||||
try {
|
||||
CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT);
|
||||
acknowledged = createIndexResponse.isAcknowledged();
|
||||
} catch (IOException e) {
|
||||
log.error("create es index exception", e);
|
||||
}
|
||||
if (acknowledged) {
|
||||
log.info("create es index success");
|
||||
isIndexExist.set(true);
|
||||
} else {
|
||||
log.error("create es index fail");
|
||||
throw new RuntimeException("cannot auto create thread-pool state es index");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getType() {
|
||||
return "es";
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cn.hippo4j.monitor.es.model;
|
||||
|
||||
import cn.hippo4j.common.model.ThreadPoolRunStateInfo;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
/**
|
||||
* Create by yuyang
|
||||
* 2022/8/4 17:17
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
public class EsThreadPoolRunStateInfo extends ThreadPoolRunStateInfo {
|
||||
|
||||
private String Id;
|
||||
|
||||
private String applicationName;
|
||||
|
||||
}
|
||||
@ -0,0 +1,78 @@
|
||||
{
|
||||
"_doc": {
|
||||
"properties": {
|
||||
"activeSize": {
|
||||
"type": "long"
|
||||
},
|
||||
"applicationName": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"clientLastRefreshTime": {
|
||||
"type": "text"
|
||||
},
|
||||
"completedTaskCount": {
|
||||
"type": "long"
|
||||
},
|
||||
"coreSize": {
|
||||
"type": "long"
|
||||
},
|
||||
"currentLoad": {
|
||||
"type": "text"
|
||||
},
|
||||
"freeMemory": {
|
||||
"type": "text"
|
||||
},
|
||||
"host": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"id": {
|
||||
"type": "text"
|
||||
},
|
||||
"largestPoolSize": {
|
||||
"type": "long"
|
||||
},
|
||||
"maximumSize": {
|
||||
"type": "long"
|
||||
},
|
||||
"memoryProportion": {
|
||||
"type": "text"
|
||||
},
|
||||
"peakLoad": {
|
||||
"type": "text"
|
||||
},
|
||||
"poolSize": {
|
||||
"type": "long"
|
||||
},
|
||||
"queueCapacity": {
|
||||
"type": "long"
|
||||
},
|
||||
"queueRemainingCapacity": {
|
||||
"type": "long"
|
||||
},
|
||||
"queueSize": {
|
||||
"type": "long"
|
||||
},
|
||||
"queueType": {
|
||||
"type": "text"
|
||||
},
|
||||
"rejectCount": {
|
||||
"type": "long"
|
||||
},
|
||||
"rejectedName": {
|
||||
"type": "text"
|
||||
},
|
||||
"simpleCurrentLoad": {
|
||||
"type": "long"
|
||||
},
|
||||
"simplePeakLoad": {
|
||||
"type": "long"
|
||||
},
|
||||
"timestamp": {
|
||||
"type": "date"
|
||||
},
|
||||
"tpId": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cn.hippo4j.core.springboot.starter.config.condition;
|
||||
|
||||
import cn.hippo4j.common.toolkit.StringUtil;
|
||||
import cn.hippo4j.core.springboot.starter.config.BootstrapCoreProperties;
|
||||
import com.example.monitor.base.MonitorTypeEnum;
|
||||
import org.springframework.context.annotation.Condition;
|
||||
import org.springframework.context.annotation.ConditionContext;
|
||||
import org.springframework.core.type.AnnotatedTypeMetadata;
|
||||
|
||||
/**
|
||||
* Prometheus monitor condition.
|
||||
*/
|
||||
public class EsMonitorCondition implements Condition {
|
||||
|
||||
@Override
|
||||
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
|
||||
String collectType = context.getEnvironment().getProperty(BootstrapCoreProperties.PREFIX + ".collect-type", "");
|
||||
return StringUtil.isNotEmpty(collectType) && collectType.contains(MonitorTypeEnum.ES.name().toLowerCase()) ? true : false;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue