feat: support lossless config from console & support warmup (#1435)

pull/1447/head
shedfreewu 4 weeks ago committed by GitHub
parent ee94dc5121
commit 22b2300d5f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -34,3 +34,4 @@
- [fix: fix PolarisCircuitBreakerConfiguration not clear when gateway invoke by wildcard apis](https://github.com/Tencent/spring-cloud-tencent/pull/1418)
- [fix:fix actuator name warning from #1428 .](https://github.com/Tencent/spring-cloud-tencent/pull/1429)
- [feat:upgrade api circuit breaker.](https://github.com/Tencent/spring-cloud-tencent/pull/1438)
- [feat: support lossless config from console & support warmup.](https://github.com/Tencent/spring-cloud-tencent/pull/1435)

@ -89,7 +89,7 @@
<properties>
<!-- Project revision -->
<revision>2.0.0.0-2022.0.5-RC2</revision>
<revision>2.0.0.0-2022.0.5-SNAPSHOT</revision>
<!-- Spring Framework -->
<spring.framework.version>6.0.22</spring.framework.version>

@ -57,7 +57,7 @@ public class PolarisLoadBalancerClientConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(value = "spring.cloud.polaris.loadbalancer.strategy", havingValue = "roundRobin", matchIfMissing = true)
@ConditionalOnProperty(value = "spring.cloud.polaris.loadbalancer.strategy", havingValue = "roundRobin")
public ReactorLoadBalancer<ServiceInstance> roundRobinLoadBalancer(Environment environment,
LoadBalancerClientFactory loadBalancerClientFactory) {
String name = environment.getProperty(LoadBalancerClientFactory.PROPERTY_NAME);
@ -97,7 +97,7 @@ public class PolarisLoadBalancerClientConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(value = "spring.cloud.polaris.loadbalancer.strategy", havingValue = "polarisWeightedRoundRobin")
@ConditionalOnProperty(value = "spring.cloud.polaris.loadbalancer.strategy", havingValue = "polarisWeightedRoundRobin", matchIfMissing = true)
public ReactorLoadBalancer<ServiceInstance> polarisWeightedRoundRobinLoadBalancer(Environment environment,
LoadBalancerClientFactory loadBalancerClientFactory, PolarisSDKContextManager polarisSDKContextManager) {
String name = environment.getProperty(LoadBalancerClientFactory.PROPERTY_NAME);

@ -19,6 +19,7 @@ package com.tencent.cloud.plugin.lossless.config;
import com.tencent.cloud.polaris.context.PolarisSDKContextManager;
import com.tencent.polaris.api.config.provider.LosslessConfig;
import com.tencent.polaris.specification.api.v1.traffic.manage.LosslessProto;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -35,14 +36,23 @@ import static org.assertj.core.api.Assertions.assertThat;
*/
public class LosslessConfigModifierTest {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
private final ApplicationContextRunner delayRegisterContextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(TestApplication.class))
.withPropertyValues("spring.cloud.nacos.discovery.enabled=false")
.withPropertyValues("spring.cloud.polaris.enabled=true")
.withPropertyValues("spring.cloud.polaris.lossless.enabled=true")
.withPropertyValues("spring.cloud.polaris.lossless.port=20000")
.withPropertyValues("spring.cloud.polaris.lossless.healthCheckPath=/xxx")
.withPropertyValues("spring.cloud.polaris.lossless.delayRegisterInterval=10")
.withPropertyValues("spring.application.name=test")
.withPropertyValues("spring.cloud.gateway.enabled=false");
private final ApplicationContextRunner healthCheckContextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(TestApplication.class))
.withPropertyValues("spring.cloud.nacos.discovery.enabled=false")
.withPropertyValues("spring.cloud.polaris.enabled=true")
.withPropertyValues("spring.cloud.polaris.lossless.enabled=true")
.withPropertyValues("spring.cloud.polaris.lossless.port=20000")
.withPropertyValues("spring.cloud.polaris.lossless.healthCheckPath=/xxx")
.withPropertyValues("spring.cloud.polaris.lossless.healthCheckInterval=5")
.withPropertyValues("spring.application.name=test")
.withPropertyValues("spring.cloud.gateway.enabled=false");
@ -60,15 +70,29 @@ public class LosslessConfigModifierTest {
}
@Test
void testModify() {
contextRunner.run(context -> {
void testDelayRegister() {
delayRegisterContextRunner.run(context -> {
PolarisSDKContextManager polarisSDKContextManager = context.getBean(PolarisSDKContextManager.class);
LosslessConfig losslessConfig = polarisSDKContextManager.getSDKContext().
getConfig().getProvider().getLossless();
assertThat(losslessConfig.getHost()).isEqualTo("0.0.0.0");
assertThat(losslessConfig.getPort()).isEqualTo(20000);
assertThat(losslessConfig.getDelayRegisterInterval()).isEqualTo(10);
assertThat(losslessConfig.getStrategy()).isEqualTo(LosslessProto.DelayRegister.DelayStrategy.DELAY_BY_TIME);
});
}
@Test
void testHealthCheck() {
healthCheckContextRunner.run(context -> {
PolarisSDKContextManager polarisSDKContextManager = context.getBean(PolarisSDKContextManager.class);
LosslessConfig losslessConfig = polarisSDKContextManager.getSDKContext().
getConfig().getProvider().getLossless();
assertThat(losslessConfig.getHost()).isEqualTo("0.0.0.0");
assertThat(losslessConfig.getPort()).isEqualTo(20000);
assertThat(losslessConfig.getHealthCheckPath()).isEqualTo("/xxx");
assertThat(losslessConfig.getHealthCheckInterval()).isEqualTo(5);
assertThat(losslessConfig.getStrategy()).isEqualTo(LosslessProto.DelayRegister.DelayStrategy.DELAY_BY_HEALTH_CHECK);
});
}

@ -0,0 +1,86 @@
/*
* Tencent is pleased to support the open source community by making Spring Cloud Tencent available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* Licensed under the BSD 3-Clause License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.tencent.cloud.plugin.lossless.config;
import com.tencent.cloud.polaris.context.PolarisSDKContextManager;
import com.tencent.polaris.api.config.consumer.WeightAdjustConfig;
import com.tencent.polaris.plugin.lossless.warmup.WarmupWeightAdjuster;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import static org.assertj.core.api.Assertions.assertThat;
/**
* Test for {@link WarmupConfigModifier}.
*
* @author Shedfree Wu
*/
public class WarmupConfigModifierTest {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(TestApplication.class))
.withPropertyValues("spring.cloud.nacos.discovery.enabled=false")
.withPropertyValues("spring.cloud.polaris.enabled=true")
.withPropertyValues("spring.cloud.polaris.warmup.enabled=true")
.withPropertyValues("spring.application.name=test")
.withPropertyValues("spring.cloud.gateway.enabled=false");
private final ApplicationContextRunner disabledContextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(TestApplication.class))
.withPropertyValues("spring.cloud.nacos.discovery.enabled=false")
.withPropertyValues("spring.cloud.polaris.enabled=true")
.withPropertyValues("spring.cloud.polaris.warmup.enabled=false")
.withPropertyValues("spring.application.name=test")
.withPropertyValues("spring.cloud.gateway.enabled=false");
@BeforeEach
void setUp() {
PolarisSDKContextManager.innerDestroy();
}
@Test
void testModify() {
contextRunner.run(context -> {
PolarisSDKContextManager polarisSDKContextManager = context.getBean(PolarisSDKContextManager.class);
WeightAdjustConfig weightAdjustConfig = polarisSDKContextManager.getSDKContext().
getConfig().getConsumer().getWeightAdjust();
assertThat(weightAdjustConfig.isEnable()).isTrue();
assertThat(weightAdjustConfig.getChain().contains(WarmupWeightAdjuster.WARMUP_WEIGHT_ADJUSTER_NAME)).isTrue();
});
}
@Test
void testDisabled() {
disabledContextRunner.run(context -> {
PolarisSDKContextManager polarisSDKContextManager = context.getBean(PolarisSDKContextManager.class);
WeightAdjustConfig weightAdjustConfig = polarisSDKContextManager.getSDKContext().
getConfig().getConsumer().getWeightAdjust();
assertThat(weightAdjustConfig.isEnable()).isTrue();
assertThat(weightAdjustConfig.getChain().contains(WarmupWeightAdjuster.WARMUP_WEIGHT_ADJUSTER_NAME)).isFalse();
});
}
@SpringBootApplication
protected static class TestApplication {
}
}

@ -17,7 +17,11 @@
package org.springframework.cloud.client.serviceregistry;
public class AutoServiceRegistrationUtils {
public final class AutoServiceRegistrationUtils {
private AutoServiceRegistrationUtils() {
}
public static void register(AbstractAutoServiceRegistration autoServiceRegistration) {
autoServiceRegistration.register();

@ -71,10 +71,10 @@
<properties>
<!-- Project revision -->
<revision>2.0.0.0-2022.0.5-RC2</revision>
<revision>2.0.0.0-2022.0.5-SNAPSHOT</revision>
<!-- Polaris SDK version -->
<polaris.version>2.0.0.0-RC2</polaris.version>
<polaris.version>2.0.0.0-SNAPSHOT</polaris.version>
<!-- Dependencies -->
<guava.version>32.0.1-jre</guava.version>

@ -30,6 +30,11 @@
<artifactId>lossless-deregister</artifactId>
</dependency>
<dependency>
<groupId>com.tencent.polaris</groupId>
<artifactId>lossless-warmup</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>

@ -22,6 +22,7 @@ import com.tencent.cloud.polaris.context.PolarisSDKContextManager;
import com.tencent.cloud.polaris.context.config.PolarisContextProperties;
import com.tencent.cloud.rpc.enhancement.transformer.RegistrationTransformer;
import com.tencent.polaris.api.pojo.BaseInstance;
import com.tencent.polaris.plugin.lossless.common.HttpLosslessActionProvider;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
@ -78,12 +79,13 @@ public class LosslessRegistryAspect {
}
// web started, get port from registration
BaseInstance instance = SpringCloudLosslessActionProvider.getBaseInstance(registration, registrationTransformer);
BaseInstance instance = getBaseInstance(registration, registrationTransformer);
Runnable registerAction = () -> executeJoinPoint(joinPoint);
SpringCloudLosslessActionProvider losslessActionProvider =
new SpringCloudLosslessActionProvider(serviceRegistry, registration, losslessProperties, registerAction);
Runnable deregisterAction = () -> serviceRegistry.deregister(registration);
HttpLosslessActionProvider losslessActionProvider =
new HttpLosslessActionProvider(registerAction, deregisterAction, registration.getPort(),
instance, polarisSDKContextManager.getSDKContext().getExtensions());
polarisSDKContextManager.getLosslessAPI().setLosslessActionProvider(instance, losslessActionProvider);
polarisSDKContextManager.getLosslessAPI().losslessRegister(instance);
@ -104,4 +106,8 @@ public class LosslessRegistryAspect {
throw new RuntimeException(e);
}
}
public static BaseInstance getBaseInstance(Registration registration, RegistrationTransformer registrationTransformer) {
return registrationTransformer.transform(registration);
}
}

@ -1,95 +0,0 @@
/*
* Tencent is pleased to support the open source community by making Spring Cloud Tencent available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* Licensed under the BSD 3-Clause License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.tencent.cloud.plugin.lossless;
import java.util.HashMap;
import java.util.Map;
import com.tencent.cloud.common.util.OkHttpUtil;
import com.tencent.cloud.plugin.lossless.config.LosslessProperties;
import com.tencent.cloud.rpc.enhancement.transformer.RegistrationTransformer;
import com.tencent.polaris.api.plugin.lossless.InstanceProperties;
import com.tencent.polaris.api.plugin.lossless.LosslessActionProvider;
import com.tencent.polaris.api.pojo.BaseInstance;
import com.tencent.polaris.api.utils.StringUtils;
import org.springframework.cloud.client.serviceregistry.Registration;
import org.springframework.cloud.client.serviceregistry.ServiceRegistry;
import org.springframework.http.HttpHeaders;
/**
* LosslessActionProvider for Spring Cloud.
*
* @author Shedfree Wu
*/
public class SpringCloudLosslessActionProvider implements LosslessActionProvider {
private ServiceRegistry<Registration> serviceRegistry;
private LosslessProperties losslessProperties;
private Runnable originalRegisterAction;
private Registration registration;
public SpringCloudLosslessActionProvider(ServiceRegistry<Registration> serviceRegistry, Registration registration,
LosslessProperties losslessProperties, Runnable originalRegisterAction) {
this.serviceRegistry = serviceRegistry;
this.registration = registration;
this.losslessProperties = losslessProperties;
this.originalRegisterAction = originalRegisterAction;
}
@Override
public String getName() {
return "spring-cloud";
}
@Override
public void doRegister(InstanceProperties instanceProperties) {
// use lambda to do original register
originalRegisterAction.run();
}
@Override
public void doDeregister() {
serviceRegistry.deregister(registration);
}
/**
* Check whether health check is enable.
* @return true: register after passing doHealthCheck, false: register after delayRegisterInterval.
*/
@Override
public boolean isEnableHealthCheck() {
return StringUtils.isNotBlank(losslessProperties.getHealthCheckPath());
}
@Override
public boolean doHealthCheck() {
Map<String, String> headers = new HashMap<>(1);
headers.put(HttpHeaders.USER_AGENT, "polaris");
return OkHttpUtil.checkUrl("localhost", registration.getPort(),
losslessProperties.getHealthCheckPath(), headers);
}
public static BaseInstance getBaseInstance(Registration registration, RegistrationTransformer registrationTransformer) {
return registrationTransformer.transform(registration);
}
}

@ -21,8 +21,10 @@ import java.util.Objects;
import com.tencent.cloud.common.constant.OrderConstant.Modifier;
import com.tencent.cloud.polaris.context.PolarisConfigModifier;
import com.tencent.polaris.api.utils.StringUtils;
import com.tencent.polaris.factory.config.ConfigurationImpl;
import com.tencent.polaris.factory.config.provider.LosslessConfigImpl;
import com.tencent.polaris.specification.api.v1.traffic.manage.LosslessProto;
/**
* Config modifier for lossless.
@ -39,17 +41,28 @@ public class LosslessConfigModifier implements PolarisConfigModifier {
@Override
public void modify(ConfigurationImpl configuration) {
LosslessConfigImpl losslessConfig = (LosslessConfigImpl) configuration.getProvider().getLossless();
if (losslessProperties.isEnabled()) {
LosslessConfigImpl losslessConfig = (LosslessConfigImpl) configuration.getProvider().getLossless();
losslessConfig.setEnable(true);
losslessConfig.setPort(losslessProperties.getPort());
if (Objects.nonNull(losslessProperties.getDelayRegisterInterval())) {
losslessConfig.setDelayRegisterInterval(losslessProperties.getDelayRegisterInterval());
}
if (Objects.nonNull(losslessProperties.getHealthCheckInterval())) {
losslessConfig.setHealthCheckInterval(losslessProperties.getHealthCheckInterval());
if (StringUtils.isNotEmpty(losslessProperties.getHealthCheckPath())) {
losslessConfig.setHealthCheckPath(losslessProperties.getHealthCheckPath());
losslessConfig.setStrategy(LosslessProto.DelayRegister.DelayStrategy.DELAY_BY_HEALTH_CHECK);
if (Objects.nonNull(losslessProperties.getHealthCheckInterval())) {
losslessConfig.setHealthCheckInterval(losslessProperties.getHealthCheckInterval());
}
}
else {
losslessConfig.setStrategy(LosslessProto.DelayRegister.DelayStrategy.DELAY_BY_TIME);
}
}
else {
losslessConfig.setEnable(false);
}
}
@Override

@ -32,7 +32,7 @@ import org.springframework.context.annotation.Configuration;
*/
@Configuration(proxyBeanMethods = false)
@ConditionalOnPolarisEnabled
@EnableConfigurationProperties(LosslessProperties.class)
@EnableConfigurationProperties({LosslessProperties.class, WarmupProperties.class})
public class LosslessPropertiesAutoConfiguration {
@Bean
@ -41,4 +41,9 @@ public class LosslessPropertiesAutoConfiguration {
return new LosslessConfigModifier(losslessProperties);
}
@Bean
@ConditionalOnMissingBean
public WarmupConfigModifier warmupConfigModifier(WarmupProperties warmupProperties) {
return new WarmupConfigModifier(warmupProperties);
}
}

@ -0,0 +1,66 @@
/*
* Tencent is pleased to support the open source community by making Spring Cloud Tencent available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* Licensed under the BSD 3-Clause License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.tencent.cloud.plugin.lossless.config;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.TreeSet;
import com.tencent.cloud.common.constant.OrderConstant.Modifier;
import com.tencent.cloud.polaris.context.PolarisConfigModifier;
import com.tencent.polaris.factory.config.ConfigurationImpl;
import com.tencent.polaris.factory.config.consumer.WeightAdjustConfigImpl;
import com.tencent.polaris.plugin.lossless.warmup.WarmupWeightAdjuster;
/**
* Config modifier for warmup.
*
* @author Shedfree Wu
*/
public class WarmupConfigModifier implements PolarisConfigModifier {
private final WarmupProperties warmupProperties;
public WarmupConfigModifier(WarmupProperties warmupProperties) {
this.warmupProperties = warmupProperties;
}
@Override
public void modify(ConfigurationImpl configuration) {
WeightAdjustConfigImpl weightAdjustConfig = (WeightAdjustConfigImpl) configuration.getConsumer().getWeightAdjust();
if (warmupProperties.isEnabled()) {
Set<String> chainSet = new TreeSet<>(
Optional.ofNullable(weightAdjustConfig.getChain()).orElse(Collections.emptyList()));
chainSet.add(WarmupWeightAdjuster.WARMUP_WEIGHT_ADJUSTER_NAME);
weightAdjustConfig.setChain(new ArrayList<>(chainSet));
}
else {
Set<String> chainSet = new TreeSet<>(
Optional.ofNullable(weightAdjustConfig.getChain()).orElse(Collections.emptyList()));
chainSet.remove(WarmupWeightAdjuster.WARMUP_WEIGHT_ADJUSTER_NAME);
weightAdjustConfig.setChain(new ArrayList<>(chainSet));
}
}
@Override
public int getOrder() {
return Modifier.LOSSLESS_ORDER;
}
}

@ -0,0 +1,35 @@
/*
* Tencent is pleased to support the open source community by making Spring Cloud Tencent available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
*
* Licensed under the BSD 3-Clause License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
*/
package com.tencent.cloud.plugin.lossless.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties("spring.cloud.polaris.warmup")
public class WarmupProperties {
private boolean enabled = false;
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
}

@ -19,10 +19,13 @@ package com.tencent.cloud.polaris.context.logging;
import com.tencent.polaris.api.utils.StringUtils;
import com.tencent.polaris.logging.LoggingConsts;
import com.tencent.polaris.logging.PolarisLogging;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.context.event.ApplicationEnvironmentPreparedEvent;
import org.springframework.boot.context.event.ApplicationFailedEvent;
import org.springframework.boot.context.logging.LoggingApplicationListener;
import org.springframework.boot.web.context.WebServerInitializedEvent;
import org.springframework.cloud.context.environment.EnvironmentChangeEvent;
import org.springframework.context.ApplicationEvent;
import org.springframework.context.event.GenericApplicationListener;
@ -37,6 +40,8 @@ import org.springframework.lang.NonNull;
*/
public class PolarisLoggingApplicationListener implements GenericApplicationListener {
private static final Logger LOG = LoggerFactory.getLogger(PolarisLoggingApplicationListener.class);
private static final int ORDER = LoggingApplicationListener.DEFAULT_ORDER + 2;
@Override
@ -47,7 +52,8 @@ public class PolarisLoggingApplicationListener implements GenericApplicationList
}
return ApplicationEnvironmentPreparedEvent.class.isAssignableFrom(type)
|| ApplicationFailedEvent.class.isAssignableFrom(type)
|| EnvironmentChangeEvent.class.isAssignableFrom(type);
|| EnvironmentChangeEvent.class.isAssignableFrom(type)
|| WebServerInitializedEvent.class.isAssignableFrom(type);
}
@Override
@ -72,7 +78,7 @@ public class PolarisLoggingApplicationListener implements GenericApplicationList
System.setProperty(LoggingConsts.LOGGING_PATH_PROPERTY, loggingPath);
}
}
LOG.info("Polaris logging configuration reloaded in {}.", applicationEvent.getClass().getSimpleName());
PolarisLogging.getInstance().loadConfiguration();
}
}

@ -39,6 +39,7 @@ public class PolarisInstanceTransformer implements InstanceTransformer {
instance.setZone(polarisServiceInstance.getPolarisInstance().getZone());
instance.setCampus(polarisServiceInstance.getPolarisInstance().getCampus());
instance.setWeight(polarisServiceInstance.getPolarisInstance().getWeight());
instance.setCreateTime(polarisServiceInstance.getPolarisInstance().getCreateTime());
if (CollectionUtils.isNotEmpty(polarisServiceInstance.getServiceMetadata())) {
instance.setServiceMetadata(polarisServiceInstance.getServiceMetadata());
}

Loading…
Cancel
Save