diff --git a/assembly/assembly-wsmaster-war/pom.xml b/assembly/assembly-wsmaster-war/pom.xml index 7dcf6aa00bfaa0b80b8b115cdc70dfd0b2e4608a..46e92fe763f8813858d4ae45ca1f7f7d3ed3ad0c 100644 --- a/assembly/assembly-wsmaster-war/pom.xml +++ b/assembly/assembly-wsmaster-war/pom.xml @@ -75,14 +75,6 @@ <groupId>io.jaegertracing</groupId> <artifactId>jaeger-tracerresolver</artifactId> </dependency> - <dependency> - <groupId>io.micrometer</groupId> - <artifactId>micrometer-core</artifactId> - </dependency> - <dependency> - <groupId>io.micrometer</groupId> - <artifactId>micrometer-registry-prometheus</artifactId> - </dependency> <dependency> <groupId>io.swagger</groupId> <artifactId>swagger-core</artifactId> @@ -187,6 +179,10 @@ <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-mail</artifactId> </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-observability</artifactId> + </dependency> <dependency> <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-schedule</artifactId> diff --git a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/MetricsOverrideBinding.java b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/MetricsOverrideBinding.java deleted file mode 100644 index 8ec7da1ad6691a1988606a79bbdd860cd413b040..0000000000000000000000000000000000000000 --- a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/MetricsOverrideBinding.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.api.deploy; - -import com.google.inject.Binder; -import com.google.inject.Module; -import io.micrometer.core.instrument.Tags; -import io.micrometer.prometheus.PrometheusMeterRegistry; -import java.util.concurrent.ExecutorService; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Singleton; -import org.eclipse.che.api.deploy.jsonrpc.CheMajorWebSocketEndpointConfiguration; -import org.eclipse.che.api.deploy.jsonrpc.CheMajorWebSocketEndpointExecutorServiceProvider; -import org.eclipse.che.api.deploy.jsonrpc.CheMinorWebSocketEndpointConfiguration; -import org.eclipse.che.api.deploy.jsonrpc.CheMinorWebSocketEndpointExecutorServiceProvider; -import org.eclipse.che.core.metrics.ExecutorServiceMetrics; - -/** - * {@link Module} that provides metered implementation for different classes. Metrics will be - * published to {@link PrometheusMeterRegistry}. - */ -public class MetricsOverrideBinding implements Module { - @Override - public void configure(Binder binder) { - binder - .bind(CheMajorWebSocketEndpointExecutorServiceProvider.class) - .to(MeteredCheMajorWebSocketEndpointExecutorServiceProvider.class); - - binder - .bind(CheMinorWebSocketEndpointExecutorServiceProvider.class) - .to(MeteredCheMinorWebSocketEndpointExecutorServiceProvider.class); - } - - @Singleton - public static class MeteredCheMajorWebSocketEndpointExecutorServiceProvider - extends CheMajorWebSocketEndpointExecutorServiceProvider { - - private final PrometheusMeterRegistry meterRegistry; - - private ExecutorService executorService; - - @Inject - public MeteredCheMajorWebSocketEndpointExecutorServiceProvider( - @Named(JSON_RPC_MAJOR_CORE_POOL_SIZE_PARAMETER_NAME) int corePoolSize, - @Named(JSON_RPC_MAJOR_MAX_POOL_SIZE_PARAMETER_NAME) int maxPoolSize, - @Named(JSON_RPC_MAJOR_QUEUE_CAPACITY_PARAMETER_NAME) int queueCapacity, - PrometheusMeterRegistry meterRegistry) { - super(corePoolSize, maxPoolSize, queueCapacity); - this.meterRegistry = meterRegistry; - } - - @Override - public synchronized ExecutorService get() { - if (executorService == null) { - executorService = - ExecutorServiceMetrics.monitor( - meterRegistry, - super.get(), - CheMajorWebSocketEndpointConfiguration.EXECUTOR_NAME, - Tags.empty()); - } - return executorService; - } - } - - @Singleton - public static class MeteredCheMinorWebSocketEndpointExecutorServiceProvider - extends CheMinorWebSocketEndpointExecutorServiceProvider { - private final PrometheusMeterRegistry meterRegistry; - - private ExecutorService executorService; - - @Inject - public MeteredCheMinorWebSocketEndpointExecutorServiceProvider( - @Named(JSON_RPC_MINOR_CORE_POOL_SIZE_PARAMETER_NAME) int corePoolSize, - @Named(JSON_RPC_MINOR_MAX_POOL_SIZE_PARAMETER_NAME) int maxPoolSize, - @Named(JSON_RPC_MINOR_QUEUE_CAPACITY_PARAMETER_NAME) int queueCapacity, - PrometheusMeterRegistry meterRegistry) { - super(corePoolSize, maxPoolSize, queueCapacity); - this.meterRegistry = meterRegistry; - } - - @Override - public synchronized ExecutorService get() { - if (executorService == null) { - - executorService = - ExecutorServiceMetrics.monitor( - meterRegistry, - super.get(), - CheMinorWebSocketEndpointConfiguration.EXECUTOR_NAME, - Tags.empty()); - } - return executorService; - } - } -} diff --git a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java index 9daa549d444a85321251242459711aad5c0feb19..380c96c95410028c502775f6bb461a4da5b3acb7 100644 --- a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java +++ b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/WsMasterModule.java @@ -68,6 +68,7 @@ import org.eclipse.che.api.workspace.server.token.MachineTokenProvider; import org.eclipse.che.api.workspace.server.wsplugins.ChePluginsApplier; import org.eclipse.che.commons.auth.token.ChainedTokenExtractor; import org.eclipse.che.commons.auth.token.RequestTokenExtractor; +import org.eclipse.che.commons.observability.deploy.ExecutorWrapperModule; import org.eclipse.che.core.db.DBTermination; import org.eclipse.che.core.db.schema.SchemaInitializer; import org.eclipse.che.core.tracing.metrics.TracingMetricsModule; @@ -260,12 +261,12 @@ public class WsMasterModule extends AbstractModule { if (Boolean.valueOf(System.getenv("CHE_METRICS_ENABLED"))) { install(new org.eclipse.che.core.metrics.MetricsModule()); install(new WsMasterMetricsModule()); - install(new MetricsOverrideBinding()); } if (Boolean.valueOf(System.getenv("CHE_TRACING_ENABLED")) && Boolean.valueOf(System.getenv("CHE_METRICS_ENABLED"))) { install(new TracingMetricsModule()); } + install(new ExecutorWrapperModule()); } private void configureSingleUserMode(Map<String, String> persistenceProperties) { diff --git a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheJsonRpcWebSocketConfigurationModule.java b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheJsonRpcWebSocketConfigurationModule.java index 6cdd0e14ec4069310f4cbe374a9e1a49127cb260..4904fb5c53ac7b6dc8123b20987d31039874d66d 100644 --- a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheJsonRpcWebSocketConfigurationModule.java +++ b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheJsonRpcWebSocketConfigurationModule.java @@ -14,23 +14,12 @@ package org.eclipse.che.api.deploy.jsonrpc; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.multibindings.Multibinder; -import com.google.inject.name.Names; -import java.util.concurrent.ExecutorService; import org.eclipse.che.api.core.jsonrpc.commons.RequestProcessorConfigurationProvider; /** Configures JSON RPC WebSocket Endpoints. */ public class CheJsonRpcWebSocketConfigurationModule implements Module { @Override public void configure(Binder binder) { - binder - .bind(ExecutorService.class) - .annotatedWith(Names.named(CheMajorWebSocketEndpointConfiguration.EXECUTOR_NAME)) - .toProvider(CheMajorWebSocketEndpointExecutorServiceProvider.class); - - binder - .bind(ExecutorService.class) - .annotatedWith(Names.named(CheMinorWebSocketEndpointConfiguration.EXECUTOR_NAME)) - .toProvider(CheMinorWebSocketEndpointExecutorServiceProvider.class); Multibinder<RequestProcessorConfigurationProvider.Configuration> configurationMultibinder = Multibinder.newSetBinder(binder, RequestProcessorConfigurationProvider.Configuration.class); diff --git a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMajorWebSocketEndpointConfiguration.java b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMajorWebSocketEndpointConfiguration.java index 187edbc5a8179d65a92dccb5408f67708d725564..e5ae721e25d4bfaaf14cb61904ab37d4cdc92c37 100644 --- a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMajorWebSocketEndpointConfiguration.java +++ b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMajorWebSocketEndpointConfiguration.java @@ -11,10 +11,17 @@ */ package org.eclipse.che.api.deploy.jsonrpc; +import static org.slf4j.LoggerFactory.getLogger; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.concurrent.ExecutorService; import javax.inject.Inject; import javax.inject.Named; import org.eclipse.che.api.core.jsonrpc.commons.RequestProcessorConfigurationProvider; +import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; +import org.eclipse.che.commons.lang.execution.ExecutorServiceBuilder; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; +import org.slf4j.Logger; /** * {@link RequestProcessorConfigurationProvider.Configuration} implementation used to configure @@ -23,13 +30,44 @@ import org.eclipse.che.api.core.jsonrpc.commons.RequestProcessorConfigurationPro public class CheMajorWebSocketEndpointConfiguration implements RequestProcessorConfigurationProvider.Configuration { + private static final Logger LOG = getLogger(CheMajorWebSocketEndpointConfiguration.class); + private final ExecutorService executor; - public static final String EXECUTOR_NAME = "che.core.jsonrpc.major_executor"; + public static final String JSON_RPC_MAJOR_CORE_POOL_SIZE_PARAMETER_NAME = + "che.core.jsonrpc.processor_core_pool_size"; + public static final String JSON_RPC_MAJOR_MAX_POOL_SIZE_PARAMETER_NAME = + "che.core.jsonrpc.processor_max_pool_size"; + public static final String JSON_RPC_MAJOR_QUEUE_CAPACITY_PARAMETER_NAME = + "che.core.jsonrpc.processor_queue_capacity"; @Inject - public CheMajorWebSocketEndpointConfiguration(@Named(EXECUTOR_NAME) ExecutorService executor) { - this.executor = executor; + public CheMajorWebSocketEndpointConfiguration( + @Named(JSON_RPC_MAJOR_CORE_POOL_SIZE_PARAMETER_NAME) int corePoolSize, + @Named(JSON_RPC_MAJOR_MAX_POOL_SIZE_PARAMETER_NAME) int maxPoolSize, + @Named(JSON_RPC_MAJOR_QUEUE_CAPACITY_PARAMETER_NAME) int queueCapacity, + ExecutorServiceWrapper wrapper) { + this.executor = + wrapper.wrap( + new ExecutorServiceBuilder() + .corePoolSize(corePoolSize) + .maxPoolSize(maxPoolSize) + .queueCapacity(queueCapacity) + .threadFactory( + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) + .setNameFormat(CheMajorWebSocketEndpoint.ENDPOINT_ID + "-%d") + .setDaemon(true) + .build()) + .rejectedExecutionHandler( + (r, executor) -> + LOG.error( + "Executor on major websocket endpoint rejected to handle the payload {}. Some important messages may be lost. Consider increasing `{}`. Now it's configured to {}", + r, + JSON_RPC_MAJOR_QUEUE_CAPACITY_PARAMETER_NAME, + queueCapacity)) + .build(), + CheMajorWebSocketEndpoint.ENDPOINT_ID); } @Override diff --git a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMajorWebSocketEndpointExecutorServiceProvider.java b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMajorWebSocketEndpointExecutorServiceProvider.java deleted file mode 100644 index 8d557f7d66d53f6f8bfb24fb6c89d738d778c392..0000000000000000000000000000000000000000 --- a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMajorWebSocketEndpointExecutorServiceProvider.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.api.deploy.jsonrpc; - -import static org.slf4j.LoggerFactory.getLogger; - -import java.util.concurrent.ExecutorService; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Singleton; -import org.eclipse.che.commons.lang.execution.ExecutorServiceProvider; -import org.slf4j.Logger; - -/** {@link ExecutorService} provider used in {@link CheMajorWebSocketEndpoint}. */ -@Singleton -public class CheMajorWebSocketEndpointExecutorServiceProvider extends ExecutorServiceProvider { - - private static final Logger LOG = getLogger(ExecutorServiceProvider.class); - - public static final String JSON_RPC_MAJOR_CORE_POOL_SIZE_PARAMETER_NAME = - "che.core.jsonrpc.processor_core_pool_size"; - public static final String JSON_RPC_MAJOR_MAX_POOL_SIZE_PARAMETER_NAME = - "che.core.jsonrpc.processor_max_pool_size"; - public static final String JSON_RPC_MAJOR_QUEUE_CAPACITY_PARAMETER_NAME = - "che.core.jsonrpc.processor_queue_capacity"; - - @Inject - public CheMajorWebSocketEndpointExecutorServiceProvider( - @Named(JSON_RPC_MAJOR_CORE_POOL_SIZE_PARAMETER_NAME) int corePoolSize, - @Named(JSON_RPC_MAJOR_MAX_POOL_SIZE_PARAMETER_NAME) int maxPoolSize, - @Named(JSON_RPC_MAJOR_QUEUE_CAPACITY_PARAMETER_NAME) int queueCapacity) { - super( - corePoolSize, - maxPoolSize, - queueCapacity, - (r, executor) -> - LOG.error( - "Executor on major websocket endpoint rejected to handle the payload {}. Some important messages may be lost. Consider increasing `{}`. Now it's configured to {}", - r, - JSON_RPC_MAJOR_QUEUE_CAPACITY_PARAMETER_NAME, - queueCapacity)); - } -} diff --git a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMinorWebSocketEndpointConfiguration.java b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMinorWebSocketEndpointConfiguration.java index 3db2a9873ac3e9ff8d2763abece435894947fa69..19296a7909688070c3b79f19065b74bb30ef3af9 100644 --- a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMinorWebSocketEndpointConfiguration.java +++ b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMinorWebSocketEndpointConfiguration.java @@ -11,10 +11,14 @@ */ package org.eclipse.che.api.deploy.jsonrpc; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.concurrent.ExecutorService; import javax.inject.Inject; import javax.inject.Named; import org.eclipse.che.api.core.jsonrpc.commons.RequestProcessorConfigurationProvider; +import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; +import org.eclipse.che.commons.lang.execution.ExecutorServiceBuilder; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; /** * {@link RequestProcessorConfigurationProvider.Configuration} implementation used to configure @@ -25,11 +29,33 @@ public class CheMinorWebSocketEndpointConfiguration private final ExecutorService executor; - public static final String EXECUTOR_NAME = "che.core.jsonrpc.minor_executor"; + public static final String JSON_RPC_MINOR_CORE_POOL_SIZE_PARAMETER_NAME = + "che.core.jsonrpc.minor_processor_core_pool_size"; + public static final String JSON_RPC_MINOR_MAX_POOL_SIZE_PARAMETER_NAME = + "che.core.jsonrpc.minor_processor_max_pool_size"; + public static final String JSON_RPC_MINOR_QUEUE_CAPACITY_PARAMETER_NAME = + "che.core.jsonrpc.minor_processor_queue_capacity"; @Inject - public CheMinorWebSocketEndpointConfiguration(@Named(EXECUTOR_NAME) ExecutorService executor) { - this.executor = executor; + public CheMinorWebSocketEndpointConfiguration( + @Named(JSON_RPC_MINOR_CORE_POOL_SIZE_PARAMETER_NAME) int corePoolSize, + @Named(JSON_RPC_MINOR_MAX_POOL_SIZE_PARAMETER_NAME) int maxPoolSize, + @Named(JSON_RPC_MINOR_QUEUE_CAPACITY_PARAMETER_NAME) int queueCapacity, + ExecutorServiceWrapper wrapper) { + this.executor = + wrapper.wrap( + new ExecutorServiceBuilder() + .corePoolSize(corePoolSize) + .maxPoolSize(maxPoolSize) + .queueCapacity(queueCapacity) + .threadFactory( + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) + .setNameFormat(CheMinorWebSocketEndpoint.ENDPOINT_ID + "-%d") + .setDaemon(true) + .build()) + .build(), + CheMinorWebSocketEndpoint.ENDPOINT_ID); } @Override diff --git a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMinorWebSocketEndpointExecutorServiceProvider.java b/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMinorWebSocketEndpointExecutorServiceProvider.java deleted file mode 100644 index e7ca07de922e463da01123a1b9c1854b13776894..0000000000000000000000000000000000000000 --- a/assembly/assembly-wsmaster-war/src/main/java/org/eclipse/che/api/deploy/jsonrpc/CheMinorWebSocketEndpointExecutorServiceProvider.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.api.deploy.jsonrpc; - -import java.util.concurrent.ExecutorService; -import javax.inject.Inject; -import javax.inject.Named; -import javax.inject.Singleton; -import org.eclipse.che.commons.lang.execution.ExecutorServiceProvider; - -/** * {@link ExecutorService} provider used in {@link CheMinorWebSocketEndpoint}. */ -@Singleton -public class CheMinorWebSocketEndpointExecutorServiceProvider extends ExecutorServiceProvider { - - public static final String JSON_RPC_MINOR_CORE_POOL_SIZE_PARAMETER_NAME = - "che.core.jsonrpc.minor_processor_core_pool_size"; - public static final String JSON_RPC_MINOR_MAX_POOL_SIZE_PARAMETER_NAME = - "che.core.jsonrpc.minor_processor_max_pool_size"; - public static final String JSON_RPC_MINOR_QUEUE_CAPACITY_PARAMETER_NAME = - "che.core.jsonrpc.minor_processor_queue_capacity"; - - @Inject - public CheMinorWebSocketEndpointExecutorServiceProvider( - @Named(JSON_RPC_MINOR_CORE_POOL_SIZE_PARAMETER_NAME) int corePoolSize, - @Named(JSON_RPC_MINOR_MAX_POOL_SIZE_PARAMETER_NAME) int maxPoolSize, - @Named(JSON_RPC_MINOR_QUEUE_CAPACITY_PARAMETER_NAME) int queueCapacity) { - - super(corePoolSize, maxPoolSize, queueCapacity); - } -} diff --git a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties index 6fc4656f9a75e721e724260de54dc39e05caa480..d0e4f339b774d35f98c8853d60a35b6ae0ab36cc 100644 --- a/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties +++ b/assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties @@ -522,7 +522,6 @@ che.core.jsonrpc.processor_max_pool_size=50 # Initial json processing pool. Minimum number of threads that used to process major JSON RPC messages. che.core.jsonrpc.processor_core_pool_size=5 # Configuration of queue used to process Json RPC messages. -# org.eclipse.che.commons.lang.execution.ExecutorServiceProvider contains more information about this parameter che.core.jsonrpc.processor_queue_capacity=100000 ## Configuration of major "/websocket-minor" endpoint @@ -532,7 +531,6 @@ che.core.jsonrpc.minor_processor_max_pool_size=100 # Initial json processing pool. Minimum number of threads that used to process minor JSON RPC messages. che.core.jsonrpc.minor_processor_core_pool_size=15 # Configuration of queue used to process Json RPC messages. -# org.eclipse.che.commons.lang.execution.ExecutorServiceProvider contains more information about this parameter che.core.jsonrpc.minor_processor_queue_capacity=10000 ## Port the the http server endpoint that would be exposed with Prometheus metrics diff --git a/assembly/assembly-wsmaster-war/src/test/java/org/eclipse/che/MetricsOverrideBindingTest.java b/assembly/assembly-wsmaster-war/src/test/java/org/eclipse/che/MetricsOverrideBindingTest.java deleted file mode 100644 index dffe9ac8cdb823914b243f14e9ada62db3880cf2..0000000000000000000000000000000000000000 --- a/assembly/assembly-wsmaster-war/src/test/java/org/eclipse/che/MetricsOverrideBindingTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che; - -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.name.Names; -import io.micrometer.core.instrument.internal.TimedExecutorService; -import java.util.concurrent.ExecutorService; -import org.eclipse.che.api.deploy.jsonrpc.CheJsonRpcWebSocketConfigurationModule; -import org.eclipse.che.api.deploy.jsonrpc.CheMajorWebSocketEndpointConfiguration; -import org.eclipse.che.core.metrics.MetricsModule; -import org.testng.Assert; -import org.testng.annotations.Test; - -public class MetricsOverrideBindingTest { - - @Test - public void shouldInjectMettered() { - Injector injector = - Guice.createInjector( - new CheJsonRpcWebSocketConfigurationModule(), - new org.eclipse.che.api.deploy.MetricsOverrideBinding(), - new MetricsModule(), - new Module() { - @Override - public void configure(Binder binder) { - binder - .bindConstant() - .annotatedWith(Names.named("che.core.jsonrpc.processor_max_pool_size")) - .to(100); - binder - .bindConstant() - .annotatedWith(Names.named("che.core.jsonrpc.processor_core_pool_size")) - .to(4); - binder - .bindConstant() - .annotatedWith(Names.named("che.core.jsonrpc.minor_processor_max_pool_size")) - .to(100); - binder - .bindConstant() - .annotatedWith(Names.named("che.core.jsonrpc.minor_processor_core_pool_size")) - .to(3); - binder - .bindConstant() - .annotatedWith(Names.named("che.core.jsonrpc.processor_queue_capacity")) - .to(100); - binder - .bindConstant() - .annotatedWith(Names.named("che.core.jsonrpc.minor_processor_queue_capacity")) - .to(100); - - binder.bindConstant().annotatedWith(Names.named("che.metrics.port")).to(100); - } - }); - CheMajorWebSocketEndpointConfiguration configuration = - injector.getInstance(CheMajorWebSocketEndpointConfiguration.class); - ExecutorService exc = configuration.getExecutorService(); - Assert.assertTrue(exc instanceof TimedExecutorService); - } -} diff --git a/core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/ExecutorServiceMetrics.java b/core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/ExecutorServiceMetrics.java deleted file mode 100644 index b59428d49910b2400ae7ba6c0277ad3fdd99ce74..0000000000000000000000000000000000000000 --- a/core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/ExecutorServiceMetrics.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.core.metrics; - -import io.micrometer.core.instrument.FunctionCounter; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import io.micrometer.core.instrument.Tags; -import io.micrometer.core.instrument.binder.MeterBinder; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionHandler; -import java.util.concurrent.ThreadPoolExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utility class that is adding monitoring rejected messages ability to standard {@link - * io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics} metrics - */ -public class ExecutorServiceMetrics implements MeterBinder { - - private static final Logger LOG = LoggerFactory.getLogger(ExecutorServiceMetrics.class); - - private final Tags tags; - - private final RejectedExecutionHandlerWrapper rejectedExecutionHandlerWrapper; - - public ExecutorServiceMetrics( - RejectedExecutionHandlerWrapper rejectedExecutionHandlerWrapper, - String executorServiceName, - Iterable<Tag> tags) { - this.rejectedExecutionHandlerWrapper = rejectedExecutionHandlerWrapper; - this.tags = Tags.concat(tags, "name", executorServiceName); - } - - public static ExecutorService monitor( - MeterRegistry registry, ExecutorService executor, String executorName, Iterable<Tag> tags) { - - if (executor instanceof ThreadPoolExecutor) { - LOG.debug("Adding rejection monitoring for {} {}", executor, executorName); - monitorRejections(registry, (ThreadPoolExecutor) executor, executorName, tags); - } - return io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics.monitor( - registry, executor, executorName, tags); - } - - public static ExecutorService monitorRejections( - MeterRegistry registry, - ThreadPoolExecutor executor, - String executorName, - Iterable<Tag> tags) { - - RejectedExecutionHandlerWrapper rejectedExecutionHandler = - new RejectedExecutionHandlerWrapper(executor.getRejectedExecutionHandler()); - executor.setRejectedExecutionHandler(rejectedExecutionHandler); - new ExecutorServiceMetrics(rejectedExecutionHandler, executorName, tags).bindTo(registry); - return io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics.monitor( - registry, executor, executorName, tags); - } - - @Override - public void bindTo(MeterRegistry registry) { - FunctionCounter.builder( - "executor.rejected", - rejectedExecutionHandlerWrapper, - RejectedExecutionHandlerWrapper::getCounter) - .tags(tags) - .description("The total number of tasks that have been rejected for execution") - .baseUnit("tasks") - .register(registry); - } - - private static class RejectedExecutionHandlerWrapper - implements java.util.concurrent.RejectedExecutionHandler { - - private final RejectedExecutionHandler handler; - private volatile long counter; - - private RejectedExecutionHandlerWrapper(RejectedExecutionHandler handler) { - this.handler = handler; - } - - @Override - public String toString() { - return "RejectedExecutionHandlerWrapper{" - + "handler=" - + handler - + ", counter=" - + counter - + '}'; - } - - @Override - public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { - counter++; - if (handler != null) { - handler.rejectedExecution(r, executor); - } - } - - public long getCounter() { - return this.counter; - } - } -} diff --git a/core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/MetricsModule.java b/core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/MetricsModule.java index fd8af67583357b681c8873b9f54a69a43459b205..dca0e341ce77702eeb2518d6d061f2c0da5ed712 100644 --- a/core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/MetricsModule.java +++ b/core/che-core-metrics-core/src/main/java/org/eclipse/che/core/metrics/MetricsModule.java @@ -16,6 +16,7 @@ import com.google.inject.AbstractModule; import com.google.inject.multibindings.Multibinder; import io.github.mweirauch.micrometer.jvm.extras.ProcessMemoryMetrics; import io.github.mweirauch.micrometer.jvm.extras.ProcessThreadMetrics; +import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.MeterBinder; import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; @@ -38,6 +39,7 @@ public class MetricsModule extends AbstractModule { bind(PrometheusMeterRegistry.class) .toProvider(PrometheusMeterRegistryProvider.class) .asEagerSingleton(); + bind(MeterRegistry.class).to(PrometheusMeterRegistry.class); Multibinder<MeterBinder> meterMultibinder = Multibinder.newSetBinder(binder(), MeterBinder.class); diff --git a/core/commons/che-core-commons-lang/pom.xml b/core/commons/che-core-commons-lang/pom.xml index dd5a5bbaf7cc011d69229355e2c99b9875011a25..354205ef973f5c780777263a6553fa2e51d0adcc 100644 --- a/core/commons/che-core-commons-lang/pom.xml +++ b/core/commons/che-core-commons-lang/pom.xml @@ -30,10 +30,6 @@ <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </dependency> - <dependency> - <groupId>javax.inject</groupId> - <artifactId>javax.inject</artifactId> - </dependency> <dependency> <groupId>javax.ws.rs</groupId> <artifactId>javax.ws.rs-api</artifactId> diff --git a/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/execution/ExecutorServiceBuilder.java b/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/execution/ExecutorServiceBuilder.java new file mode 100644 index 0000000000000000000000000000000000000000..980e93f32b0b282af38ade0ea7bc719d2abb0243 --- /dev/null +++ b/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/execution/ExecutorServiceBuilder.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.lang.execution; + +import java.time.Duration; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Helper class to build {@link ExecutorService} from parts like corePoolSize, maxPoolSize, + * threadFactory, etc. + * + * @author Sergii Kabashniuk + */ +public class ExecutorServiceBuilder { + + private int corePoolSize; + private int maxPoolSize; + private boolean allowCoreThreadTimeOut; + private Duration keepAliveTime; + private BlockingQueue<Runnable> workQueue; + private ThreadFactory threadFactory; + private RejectedExecutionHandler handler; + + public ExecutorServiceBuilder() { + this.corePoolSize = 0; + this.maxPoolSize = 1; + this.allowCoreThreadTimeOut = false; + this.keepAliveTime = Duration.ofSeconds(60); + this.workQueue = new LinkedBlockingQueue<>(); + this.threadFactory = Executors.defaultThreadFactory(); + this.handler = new ThreadPoolExecutor.AbortPolicy(); + } + + /** + * @param corePoolSize - configure corePoolSize the number of threads to keep in the pool, even if + * they are idle, unless {@code allowCoreThreadTimeOut} is set. + */ + public ExecutorServiceBuilder corePoolSize(int corePoolSize) { + this.corePoolSize = corePoolSize; + return this; + } + + /** @param maxPoolSize - configure the maximum number of threads to allow in the pool. */ + public ExecutorServiceBuilder maxPoolSize(int maxPoolSize) { + if (maxPoolSize < corePoolSize) { + throw new IllegalArgumentException("maxPoolSize has to be grater then corePoolSize"); + } + this.maxPoolSize = maxPoolSize; + return this; + } + + /** @param allowCoreThreadTimeOut - allow core threads to time out an terminate. */ + public ExecutorServiceBuilder allowCoreThreadTimeOut(boolean allowCoreThreadTimeOut) { + this.allowCoreThreadTimeOut = allowCoreThreadTimeOut; + return this; + } + + /** @param time - configure keepAliveTime parameter of {@code ThreadPoolExecutor} */ + public ExecutorServiceBuilder keepAliveTime(Duration time) { + this.keepAliveTime = time; + return this; + } + + /** + * @param workQueue - configure the type of the task queue that would be used in {@code + * ThreadPoolExecutor} + */ + public ExecutorServiceBuilder workQueue(BlockingQueue<Runnable> workQueue) { + this.workQueue = workQueue; + return this; + } + + /** + * @param queueCapacity - configure {@code LinkedBlockingQueue} with queueCapacity capacity to be + * used in {@code ThreadPoolExecutor} + */ + public ExecutorServiceBuilder queueCapacity(int queueCapacity) { + this.workQueue = new LinkedBlockingQueue<>(queueCapacity); + return this; + } + + /** + * @param handler - configure instance of {@code RejectedExecutionHandler} to be used in {@code + * ThreadPoolExecutor} + */ + public ExecutorServiceBuilder rejectedExecutionHandler(RejectedExecutionHandler handler) { + this.handler = handler; + return this; + } + + /** + * @param threadFactory - configure instance of {@code ThreadFactory} to be used in {@code + * ThreadPoolExecutor} + */ + public ExecutorServiceBuilder threadFactory(ThreadFactory threadFactory) { + this.threadFactory = threadFactory; + return this; + } + + public ExecutorService build() { + + final ThreadPoolExecutor executor = + new ThreadPoolExecutor( + corePoolSize, + maxPoolSize, + keepAliveTime.toMillis(), + TimeUnit.MILLISECONDS, + workQueue, + threadFactory, + handler); + executor.allowCoreThreadTimeOut(allowCoreThreadTimeOut); + + return executor; + } +} diff --git a/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/execution/ExecutorServiceProvider.java b/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/execution/ExecutorServiceProvider.java deleted file mode 100644 index 092a4024c8ccc64ccb3a76c524258b83c18c432c..0000000000000000000000000000000000000000 --- a/core/commons/che-core-commons-lang/src/main/java/org/eclipse/che/commons/lang/execution/ExecutorServiceProvider.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.commons.lang.execution; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.slf4j.LoggerFactory.getLogger; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionHandler; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import javax.inject.Provider; -import javax.inject.Singleton; -import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; -import org.slf4j.Logger; - -/** - * Configurable {@link Provider} of {@link ExecutorService}. - * - * <p>Allow to configure corePoolSize, maximumPoolSize, queueCapacity. It uses two different - * implementation of Queue. if queueCapacity > 0 then this is capacity of {@link - * LinkedBlockingQueue} if <=0 then {@link SynchronousQueue} is used. - * - * <p>Implementation add {@link java.util.concurrent.RejectedExecutionHandler} that is printing - * rejected message to the LOG.error. This can happen in case if there is no available Thread in - * ThreadPool and there is no capacity in the queue. - * - * @author Sergii Kabashniuk - */ -@Singleton -public class ExecutorServiceProvider implements Provider<ExecutorService> { - - private static final Logger LOG = getLogger(ExecutorServiceProvider.class); - - private final ThreadPoolExecutor executor; - /** - * @param corePoolSize - corePoolSize of ThreadPoolExecutor - * @param maxPoolSize - maximumPoolSize of ThreadPoolExecutor - * @param queueCapacity - queue capacity. if > 0 then this is capacity of {@link - * LinkedBlockingQueue} if <=0 then {@link SynchronousQueue} are used. - * @param rejectedExecutionHandler - {@link RejectedExecutionHandler} of ThreadPoolExecutor - */ - public ExecutorServiceProvider( - int corePoolSize, - int maxPoolSize, - int queueCapacity, - RejectedExecutionHandler rejectedExecutionHandler) { - ThreadFactory factory = - new ThreadFactoryBuilder() - .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) - .setNameFormat(this.getClass().getSimpleName() + "-%d") - .setDaemon(true) - .build(); - - executor = - new ThreadPoolExecutor( - corePoolSize, - maxPoolSize, - 60L, - SECONDS, - queueCapacity > 0 ? new LinkedBlockingQueue<>(queueCapacity) : new SynchronousQueue<>(), - factory); - executor.setRejectedExecutionHandler(rejectedExecutionHandler); - executor.prestartCoreThread(); - } - - /** - * @param corePoolSize - corePoolSize of ThreadPoolExecutor - * @param maxPoolSize - maximumPoolSize of ThreadPoolExecutor - * @param queueCapacity - queue capacity. if > 0 then this is capacity of {@link - * LinkedBlockingQueue} if <=0 then {@link SynchronousQueue} are used. - */ - public ExecutorServiceProvider(int corePoolSize, int maxPoolSize, int queueCapacity) { - this( - corePoolSize, - maxPoolSize, - queueCapacity, - (r, e) -> LOG.warn("Executor rejected to handle the payload {}", r)); - } - - @Override - public ExecutorService get() { - return executor; - } -} diff --git a/core/commons/che-core-commons-lang/src/test/java/org/eclipse/che/commons/lang/execution/ExecutorServiceBuilderTest.java b/core/commons/che-core-commons-lang/src/test/java/org/eclipse/che/commons/lang/execution/ExecutorServiceBuilderTest.java new file mode 100644 index 0000000000000000000000000000000000000000..addbc25039e7322b005a8b91eb81b01d4fbc32ab --- /dev/null +++ b/core/commons/che-core-commons-lang/src/test/java/org/eclipse/che/commons/lang/execution/ExecutorServiceBuilderTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.lang.execution; + +import static org.testng.Assert.*; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; +import org.testng.annotations.Test; + +public class ExecutorServiceBuilderTest { + ThreadFactory threadFactory = + new ThreadFactoryBuilder() + .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) + .setNameFormat(ExecutorServiceBuilderTest.class + "-%d") + .setDaemon(true) + .build(); + + @Test + public void testBuild() { + + ExecutorService executorService = + new ExecutorServiceBuilder() + .corePoolSize(6) + .maxPoolSize(12) + .queueCapacity(5000) + .threadFactory(threadFactory) + .build(); + + assertTrue(executorService instanceof ThreadPoolExecutor); + + ThreadPoolExecutor threadPoolExecutorService = (ThreadPoolExecutor) executorService; + assertEquals(threadPoolExecutorService.getCorePoolSize(), 6); + assertEquals(threadPoolExecutorService.getMaximumPoolSize(), 12); + assertEquals(threadPoolExecutorService.getThreadFactory(), threadFactory); + } + + @Test( + expectedExceptions = IllegalArgumentException.class, + expectedExceptionsMessageRegExp = "maxPoolSize has to be grater then corePoolSize") + public void testSetMaxLowerThenCore() { + new ExecutorServiceBuilder().corePoolSize(6).maxPoolSize(1).build(); + } +} diff --git a/core/commons/che-core-commons-lang/src/test/java/org/eclipse/che/commons/lang/execution/ExecutorServiceProviderTest.java b/core/commons/che-core-commons-lang/src/test/java/org/eclipse/che/commons/lang/execution/ExecutorServiceProviderTest.java deleted file mode 100644 index fd0ad89d862fdeca0ba2c83a128c350f61400942..0000000000000000000000000000000000000000 --- a/core/commons/che-core-commons-lang/src/test/java/org/eclipse/che/commons/lang/execution/ExecutorServiceProviderTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2012-2018 Red Hat, Inc. - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * - * Contributors: - * Red Hat, Inc. - initial API and implementation - */ -package org.eclipse.che.commons.lang.execution; - -import static org.testng.Assert.*; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import org.testng.Assert; -import org.testng.annotations.Test; - -public class ExecutorServiceProviderTest { - @Test - public void shouldProvideExecutorService() { - // given - ExecutorServiceProvider executorServiceProvider = new ExecutorServiceProvider(10, 10, 10); - // when - ExecutorService executorService = executorServiceProvider.get(); - // then - Assert.assertNotNull(executorService); - } - - @Test - public void shouldProvideExecutorServiceWithSynchronousQueue() { - // given - ExecutorServiceProvider executorServiceProvider = new ExecutorServiceProvider(10, 10, 0); - - // when - ExecutorService executorService = executorServiceProvider.get(); - - // then - assertTrue(executorService instanceof ThreadPoolExecutor); - assertTrue(((ThreadPoolExecutor) executorService).getQueue() instanceof SynchronousQueue); - } - - @Test - public void shouldProvideExecutorServiceWithLinkedBlockingQueue() { - // given - ExecutorServiceProvider executorServiceProvider = new ExecutorServiceProvider(10, 10, 10); - - // when - ExecutorService executorService = executorServiceProvider.get(); - - // then - assertTrue(executorService instanceof ThreadPoolExecutor); - assertTrue(((ThreadPoolExecutor) executorService).getQueue() instanceof LinkedBlockingQueue); - } - - @Test - public void shouldProvideExecutorServiceWithCorePoolSize() { - // given - ExecutorServiceProvider executorServiceProvider = - new ExecutorServiceProvider(100500, 100501, 10); - - // when - ExecutorService executorService = executorServiceProvider.get(); - - // then - assertTrue(executorService instanceof ThreadPoolExecutor); - assertEquals(((ThreadPoolExecutor) executorService).getCorePoolSize(), 100500); - } - - @Test - public void shouldProvideExecutorServiceWithMaxPoolSize() { - // given - ExecutorServiceProvider executorServiceProvider = new ExecutorServiceProvider(10, 28, 10); - - // when - ExecutorService executorService = executorServiceProvider.get(); - - // then - assertTrue(executorService instanceof ThreadPoolExecutor); - assertEquals(((ThreadPoolExecutor) executorService).getMaximumPoolSize(), 28); - } -} diff --git a/core/commons/che-core-commons-mail/pom.xml b/core/commons/che-core-commons-mail/pom.xml index 7cbfdd4e90f4830d3241ced0cbbf26aa05ca97c3..4001ac791e83777d0178cb1130b38099c08bc5ad 100644 --- a/core/commons/che-core-commons-mail/pom.xml +++ b/core/commons/che-core-commons-mail/pom.xml @@ -59,6 +59,10 @@ <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-lang</artifactId> </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-observability</artifactId> + </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> diff --git a/core/commons/che-core-commons-mail/src/main/java/org/eclipse/che/mail/MailSender.java b/core/commons/che-core-commons-mail/src/main/java/org/eclipse/che/mail/MailSender.java index b50662dca43f09ef1e95d2274f245f7efb268cb1..56bd501f6695879e6cc1520f8c86562eb8bb5d1a 100644 --- a/core/commons/che-core-commons-mail/src/main/java/org/eclipse/che/mail/MailSender.java +++ b/core/commons/che-core-commons-mail/src/main/java/org/eclipse/che/mail/MailSender.java @@ -33,6 +33,7 @@ import javax.mail.internet.MimeMessage; import javax.mail.internet.MimeMultipart; import org.apache.commons.io.FileUtils; import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,16 +51,18 @@ public class MailSender { private final MailSessionProvider mailSessionProvider; @Inject - public MailSender(MailSessionProvider mailSessionProvider) { + public MailSender(MailSessionProvider mailSessionProvider, ExecutorServiceWrapper wrapper) { this.mailSessionProvider = mailSessionProvider; this.executor = - newFixedThreadPool( - 2 * Runtime.getRuntime().availableProcessors(), - new ThreadFactoryBuilder() - .setNameFormat("MailNotificationsPool-%d") - .setDaemon(false) - .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) - .build()); + wrapper.wrap( + newFixedThreadPool( + 2 * Runtime.getRuntime().availableProcessors(), + new ThreadFactoryBuilder() + .setNameFormat("MailNotificationsPool-%d") + .setDaemon(false) + .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) + .build()), + MailSender.class.getName()); } public void sendAsync(EmailBean emailBean) { diff --git a/core/commons/che-core-commons-mail/src/test/java/org/eclipse/che/mail/MailSenderTest.java b/core/commons/che-core-commons-mail/src/test/java/org/eclipse/che/mail/MailSenderTest.java index f08f77523a05a23f142b6e4ecaf9e2f969ed3d7a..0ce66956158b63b660cd2e7918b1d7f824a7fae3 100644 --- a/core/commons/che-core-commons-mail/src/test/java/org/eclipse/che/mail/MailSenderTest.java +++ b/core/commons/che-core-commons-mail/src/test/java/org/eclipse/che/mail/MailSenderTest.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Base64; import java.util.Collections; import java.util.Map; +import org.eclipse.che.commons.observability.NoopExecutorServiceWrapper; import org.testng.ITestContext; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -71,7 +72,9 @@ public class MailSenderTest { " mail.smtp.auth", "false"); - mailSender = new MailSender(new MailSessionProvider(mailConfiguration)); + mailSender = + new MailSender( + new MailSessionProvider(mailConfiguration), new NoopExecutorServiceWrapper()); } @AfterMethod diff --git a/core/commons/che-core-commons-observability/pom.xml b/core/commons/che-core-commons-observability/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..93309e8a5b6dd138bc978131dc898f799988e9d8 --- /dev/null +++ b/core/commons/che-core-commons-observability/pom.xml @@ -0,0 +1,76 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Copyright (c) 2012-2018 Red Hat, Inc. + This program and the accompanying materials are made + available under the terms of the Eclipse Public License 2.0 + which is available at https://www.eclipse.org/legal/epl-2.0/ + + SPDX-License-Identifier: EPL-2.0 + + Contributors: + Red Hat, Inc. - initial API and implementation + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>che-core-commons-parent</artifactId> + <groupId>org.eclipse.che.core</groupId> + <version>7.4.0-SNAPSHOT</version> + </parent> + <artifactId>che-core-commons-observability</artifactId> + <name>Che Core :: Commons :: Tracing and Monitoring wrapper</name> + <dependencies> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + </dependency> + <dependency> + <groupId>io.micrometer</groupId> + <artifactId>micrometer-core</artifactId> + </dependency> + <dependency> + <groupId>io.opentracing</groupId> + <artifactId>opentracing-api</artifactId> + </dependency> + <dependency> + <groupId>io.opentracing.contrib</groupId> + <artifactId>opentracing-concurrent</artifactId> + </dependency> + <dependency> + <groupId>javax.inject</groupId> + <artifactId>javax.inject</artifactId> + </dependency> + <dependency> + <groupId>javax.validation</groupId> + <artifactId>validation-api</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-lang</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-schedule</artifactId> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testng</groupId> + <artifactId>testng</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> diff --git a/core/commons/che-core-commons-observability/src/main/java/io/micrometer/core/instrument/internal/TimedCronExecutorService.java b/core/commons/che-core-commons-observability/src/main/java/io/micrometer/core/instrument/internal/TimedCronExecutorService.java new file mode 100644 index 0000000000000000000000000000000000000000..16c82fbbbae2010795679dd70b0b742ed03c71cc --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/io/micrometer/core/instrument/internal/TimedCronExecutorService.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package io.micrometer.core.instrument.internal; + +import static java.util.stream.Collectors.toList; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; +import org.eclipse.che.commons.schedule.executor.CronExpression; + +/** + * A {@link CronExecutorService} that is timed. It provides the same metrics as {@link + * io.micrometer.core.instrument.internal.TimedScheduledExecutorService} plus adds one extra counter + * {@code executor.scheduled.cron}. The counter represents the number of invocations of method + * {@link CronExecutorService#schedule(Runnable, CronExpression)} + * + * @author Sergii Kabashniuk + */ +public class TimedCronExecutorService implements CronExecutorService { + private final CronExecutorService delegate; + + private final Counter scheduledCron; + + private final MeterRegistry registry; + private final Timer executionTimer; + private final Timer idleTimer; + private final Counter scheduledOnce; + private final Counter scheduledRepetitively; + + public TimedCronExecutorService( + MeterRegistry registry, + CronExecutorService delegate, + String executorServiceName, + Iterable<Tag> tags) { + this.registry = registry; + this.delegate = delegate; + this.executionTimer = + registry.timer("executor", Tags.concat(tags, "name", executorServiceName)); + this.idleTimer = + registry.timer("executor.idle", Tags.concat(tags, "name", executorServiceName)); + this.scheduledOnce = + registry.counter("executor.scheduled.once", Tags.concat(tags, "name", executorServiceName)); + this.scheduledRepetitively = + registry.counter( + "executor.scheduled.repetitively", Tags.concat(tags, "name", executorServiceName)); + this.scheduledCron = + registry.counter("executor.scheduled.cron", Tags.concat(tags, "name", executorServiceName)); + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List<Runnable> shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + + @Override + public <T> Future<T> submit(Callable<T> task) { + return delegate.submit(wrap(task)); + } + + @Override + public <T> Future<T> submit(Runnable task, T result) { + return delegate.submit(wrap(task), result); + } + + @Override + public Future<?> submit(Runnable task) { + return delegate.submit(wrap(task)); + } + + @Override + public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) + throws InterruptedException { + return delegate.invokeAll(wrapAll(tasks)); + } + + @Override + public <T> List<Future<T>> invokeAll( + Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) + throws InterruptedException { + return delegate.invokeAll(wrapAll(tasks), timeout, unit); + } + + @Override + public <T> T invokeAny(Collection<? extends Callable<T>> tasks) + throws InterruptedException, ExecutionException { + return delegate.invokeAny(wrapAll(tasks)); + } + + @Override + public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return delegate.invokeAny(wrapAll(tasks), timeout, unit); + } + + @Override + public void execute(Runnable command) { + delegate.execute(wrap(command)); + } + + @Override + public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) { + scheduledOnce.increment(); + return delegate.schedule(executionTimer.wrap(command), delay, unit); + } + + @Override + public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) { + scheduledOnce.increment(); + return delegate.schedule(executionTimer.wrap(callable), delay, unit); + } + + @Override + public ScheduledFuture<?> scheduleAtFixedRate( + Runnable command, long initialDelay, long period, TimeUnit unit) { + scheduledRepetitively.increment(); + return delegate.scheduleAtFixedRate(executionTimer.wrap(command), initialDelay, period, unit); + } + + @Override + public ScheduledFuture<?> scheduleWithFixedDelay( + Runnable command, long initialDelay, long delay, TimeUnit unit) { + scheduledRepetitively.increment(); + return delegate.scheduleWithFixedDelay(executionTimer.wrap(command), initialDelay, delay, unit); + } + + private Runnable wrap(Runnable task) { + return new TimedRunnable(registry, executionTimer, idleTimer, task); + } + + private <T> Callable<T> wrap(Callable<T> task) { + return new TimedCallable<>(registry, executionTimer, idleTimer, task); + } + + private <T> Collection<? extends Callable<T>> wrapAll(Collection<? extends Callable<T>> tasks) { + return tasks.stream().map(this::wrap).collect(toList()); + } + + @Override + public Future<?> schedule(Runnable task, CronExpression expression) { + scheduledCron.increment(); + return delegate.schedule(executionTimer.wrap(task), expression); + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/CountedRejectedExecutionHandler.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/CountedRejectedExecutionHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..abb71ec15c877ff0c9c57fa27c0ae9b6b4000d2f --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/CountedRejectedExecutionHandler.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.BaseUnits; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * Wrapper of {@link RejectedExecutionHandler} that reports to {@link MeterRegistry} number of + * rejections. + */ +public class CountedRejectedExecutionHandler implements RejectedExecutionHandler { + + private final RejectedExecutionHandler delegate; + private final Counter counter; + + public CountedRejectedExecutionHandler( + RejectedExecutionHandler delegate, MeterRegistry registry, String name, Iterable<Tag> tags) { + this.delegate = delegate; + this.counter = + Counter.builder("executor.rejected") + .tags(Tags.concat(tags, "name", name)) + .description("The number of tasks that was not accepted for execution") + .baseUnit(BaseUnits.TASKS) + .register(registry); + } + + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + counter.increment(); + delegate.rejectedExecution(r, executor); + } + + public static void monitorRejections( + MeterRegistry registry, ThreadPoolExecutor executor, String name, Iterable<Tag> tags) { + executor.setRejectedExecutionHandler( + new CountedRejectedExecutionHandler( + executor.getRejectedExecutionHandler(), registry, name, tags)); + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/CountedThreadFactory.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/CountedThreadFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..54c226624c68605b609b070d90da49063e7a3744 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/CountedThreadFactory.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.BaseUnits; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; +import javax.validation.constraints.NotNull; + +/** A {@link ThreadFactory} that monitors the number of threads created, running and terminated. */ +public class CountedThreadFactory implements ThreadFactory { + + private final ThreadFactory delegate; + private final Counter created; + private final AtomicInteger running = new AtomicInteger(0); + private final Counter terminated; + + /** + * Wraps a {@link ThreadFactory} with an explicit name and records the number of created, running + * and terminated threads. + * + * @param delegate {@link ThreadFactory} to wrap. + * @param registry {@link MeterRegistry} that will contain the metrics. + * @param name name for this delegate. + * @param tags tags that can provide additional context. + */ + public CountedThreadFactory( + ThreadFactory delegate, MeterRegistry registry, String name, Iterable<Tag> tags) { + this.delegate = delegate; + this.created = + Counter.builder("thread.factory.created") + .tags(Tags.concat(tags, "name", name)) + .description("The approximate number of threads that create with thread factory") + .baseUnit(BaseUnits.THREADS) + .register(registry); + this.terminated = + Counter.builder("thread.factory.terminated") + .tags(Tags.concat(tags, "name", name)) + .description("The approximate number of threads that is finished execution") + .baseUnit(BaseUnits.THREADS) + .register(registry); + Gauge.builder("thread.factory.running", running, AtomicInteger::get) + .tags(Tags.concat(tags, "name", name)) + .description( + "The approximate number of threads that are started executing, but not terminated") + .baseUnit(BaseUnits.THREADS) + .register(registry); + } + + /** {@inheritDoc} */ + @Override + public Thread newThread(@NotNull Runnable runnable) { + + Thread thread = + delegate.newThread( + () -> { + running.incrementAndGet(); + try { + runnable.run(); + } finally { + running.decrementAndGet(); + terminated.increment(); + } + }); + created.increment(); + return thread; + } + + public static void monitorThreads( + MeterRegistry registry, ThreadPoolExecutor executor, String name, Iterable<Tag> tags) { + executor.setThreadFactory( + new CountedThreadFactory(executor.getThreadFactory(), registry, name, tags)); + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/ExecutorServiceWrapper.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/ExecutorServiceWrapper.java new file mode 100644 index 0000000000000000000000000000000000000000..50ef304f335aaad3a6e2b4aad2675d97e788af39 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/ExecutorServiceWrapper.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import com.google.common.annotations.Beta; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; + +/** + * Wrapper of different implemntations of {@link ExecutorService}. At this moment supported: {@link + * ExecutorService}, {@link ScheduledExecutorService} and {@link CronExecutorService}. + * + * <p>Depending on implementation and environment configuration wrapper can add to the original + * class different capabilities like monitoring or tracing. + * + * @author Sergii Kabashniuk + */ +@Beta +public interface ExecutorServiceWrapper { + + /** + * Creates wrapper for the given executor. + * + * @param executor {@link ExecutorService} that has to be wrapped. + * @param name unique name that can identify concrete instance of executor. + * @param tags key/value pairs that gives some context about provided executor. + * @return wrapped instance of given executor. + */ + ExecutorService wrap(ExecutorService executor, String name, String... tags); + + /** + * Creates wrapper for the given executor. + * + * @param executor {@link ScheduledExecutorService} that has to be wrapped. + * @param name unique name that can identify concrete instance of executor. + * @param tags key/value pairs that gives some context about provided executor. + * @return wrapped instance of given executor. + */ + ScheduledExecutorService wrap(ScheduledExecutorService executor, String name, String... tags); + + /** + * Creates wrapper for the given executor. + * + * @param executor {@link CronExecutorService} that has to be wrapped. + * @param name unique name that can identify concrete instance of executor. + * @param tags key/value pairs that gives some context about provided executor. + * @return wrapped instance of given executor. + */ + CronExecutorService wrap(CronExecutorService executor, String name, String... tags); +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/MeteredAndTracedExecutorServiceWrapper.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/MeteredAndTracedExecutorServiceWrapper.java new file mode 100644 index 0000000000000000000000000000000000000000..d0b75a38eca1629625905f97418530a804513a39 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/MeteredAndTracedExecutorServiceWrapper.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import javax.inject.Inject; +import javax.inject.Singleton; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; + +/** + * Implementation of {@code ExecutorServiceWrapper} that add all sort of monitoring and tracing + * capabilities. Monitoring allayed first, tracing second. + */ +@Singleton +public class MeteredAndTracedExecutorServiceWrapper implements ExecutorServiceWrapper { + + private final MeteredExecutorServiceWrapper meteredExecutorServiceWrapper; + private final TracedExecutorServiceWrapper tracedExecutorServiceWrapper; + + @Inject + public MeteredAndTracedExecutorServiceWrapper( + MeteredExecutorServiceWrapper meteredExecutorServiceWrapper, + TracedExecutorServiceWrapper tracedExecutorServiceWrapper) { + + this.meteredExecutorServiceWrapper = meteredExecutorServiceWrapper; + this.tracedExecutorServiceWrapper = tracedExecutorServiceWrapper; + } + + @Override + public ExecutorService wrap(ExecutorService executor, String name, String... tags) { + return tracedExecutorServiceWrapper.wrap( + meteredExecutorServiceWrapper.wrap(executor, name, tags), name, tags); + } + + @Override + public ScheduledExecutorService wrap( + ScheduledExecutorService executor, String name, String... tags) { + return tracedExecutorServiceWrapper.wrap( + meteredExecutorServiceWrapper.wrap(executor, name, tags), name, tags); + } + + @Override + public CronExecutorService wrap(CronExecutorService executor, String name, String... tags) { + return tracedExecutorServiceWrapper.wrap( + meteredExecutorServiceWrapper.wrap(executor, name, tags), name, tags); + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/MeteredExecutorServiceWrapper.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/MeteredExecutorServiceWrapper.java new file mode 100644 index 0000000000000000000000000000000000000000..3ff8fac4c1d466203103e1495209b33bcbbcf482 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/MeteredExecutorServiceWrapper.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics; +import io.micrometer.core.instrument.internal.TimedCronExecutorService; +import io.micrometer.core.lang.Nullable; +import java.lang.reflect.Field; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import javax.inject.Inject; +import javax.inject.Singleton; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of {@code ExecutorServiceWrapper} that add all sort of monitoring capabilities + * from {@code ExecutorServiceMetrics}. + * + * <p>Also in case if a provided executor is an instance of {@code ThreadPoolExecutor} it will add + * metrics provided by {@code CountedThreadFactory} and {@code CountedRejectedExecutionHandler}. In + * case if {@code ExecutorService} provided by {@code Executors} class are the instances of + * java.util.concurrent.Executors$DelegatedScheduledExecutorService or + * java.util.concurrent.Executors$FinalizableDelegatedExecutorService there would be an attempt to + * unwrap them to get underlying {@code ThreadPoolExecutor} to be able to provide {@code + * CountedThreadFactory} and {@code CountedRejectedExecutionHandler} statistics. Failed unwrapping + * attempt would be only logged, no exception would be raised and no additional statistic would be + * published. + */ +@Singleton +public class MeteredExecutorServiceWrapper implements ExecutorServiceWrapper { + private static final Logger LOG = LoggerFactory.getLogger(MeteredExecutorServiceWrapper.class); + + private final MeterRegistry meterRegistry; + + @Inject + public MeteredExecutorServiceWrapper(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + @Override + public ScheduledExecutorService wrap( + ScheduledExecutorService executor, String name, String... tags) { + + monitorThreadPoolExecutor(executor, name, tags); + return ExecutorServiceMetrics.monitor(meterRegistry, executor, name, Tags.of(tags)); + } + + @Override + public ExecutorService wrap(ExecutorService executor, String name, String... tags) { + monitorThreadPoolExecutor(executor, name, tags); + return ExecutorServiceMetrics.monitor(meterRegistry, executor, name, Tags.of(tags)); + } + + @Override + public CronExecutorService wrap(CronExecutorService executor, String name, String... tags) { + monitorThreadPoolExecutor(executor, name, tags); + new io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics( + executor, name, Tags.of(tags)) + .bindTo(meterRegistry); + + return new TimedCronExecutorService(meterRegistry, executor, name, Tags.of(tags)); + } + + private void monitorThreadPoolExecutor(ExecutorService executor, String name, String... tags) { + String className = executor.getClass().getName(); + ThreadPoolExecutor unwrappedThreadPoolExecutor = null; + if (executor instanceof ThreadPoolExecutor) { + unwrappedThreadPoolExecutor = (ThreadPoolExecutor) executor; + } else if (className.equals( + "java.util.concurrent.Executors$DelegatedScheduledExecutorService")) { + unwrappedThreadPoolExecutor = unwrapThreadPoolExecutor(executor, executor.getClass()); + } else if (className.equals( + "java.util.concurrent.Executors$FinalizableDelegatedExecutorService")) { + unwrappedThreadPoolExecutor = + unwrapThreadPoolExecutor(executor, executor.getClass().getSuperclass()); + } + if (unwrappedThreadPoolExecutor != null) { + CountedThreadFactory.monitorThreads( + meterRegistry, unwrappedThreadPoolExecutor, name, Tags.of(tags)); + CountedRejectedExecutionHandler.monitorRejections( + meterRegistry, unwrappedThreadPoolExecutor, name, Tags.of(tags)); + } + } + + /** + * Every ScheduledThreadPoolExecutor created by {@link Executors} is wrapped. Also, {@link + * Executors#newSingleThreadExecutor()} wrap a regular {@link ThreadPoolExecutor}. + */ + @Nullable + private ThreadPoolExecutor unwrapThreadPoolExecutor(ExecutorService executor, Class<?> wrapper) { + try { + Field e = wrapper.getDeclaredField("e"); + e.setAccessible(true); + return (ThreadPoolExecutor) e.get(executor); + } catch (NoSuchFieldException | IllegalAccessException e) { + LOG.error( + String.format( + "Unable to unwrap ThreadPoolExecutor from %s instance of %s." + + " CountedThreadFactory and CountedThreadFactory statistic would be omitted", + executor, wrapper), + e); + } + return null; + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/NoopExecutorServiceWrapper.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/NoopExecutorServiceWrapper.java new file mode 100644 index 0000000000000000000000000000000000000000..0b732220139697cb9855389b993619b3b53eea8f --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/NoopExecutorServiceWrapper.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import javax.inject.Singleton; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; + +/** + * Implementation of {@link ExecutorServiceWrapper} that adds nothing. All wrap methods return the + * same instance as result. + * + * @author Sergii Kabashniuk + */ +@Singleton +public class NoopExecutorServiceWrapper implements ExecutorServiceWrapper { + + @Override + public ExecutorService wrap(ExecutorService executor, String name, String... tags) { + return executor; + } + + @Override + public ScheduledExecutorService wrap( + ScheduledExecutorService executor, String name, String... tags) { + return executor; + } + + @Override + public CronExecutorService wrap(CronExecutorService executor, String name, String... tags) { + return executor; + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/ObservableThreadPullLauncher.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/ObservableThreadPullLauncher.java new file mode 100644 index 0000000000000000000000000000000000000000..27c634c15c416e8add0971abd922666ec4878704 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/ObservableThreadPullLauncher.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import javax.inject.Inject; +import javax.inject.Named; +import javax.inject.Singleton; +import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; +import org.eclipse.che.commons.schedule.executor.CronThreadPoolExecutor; +import org.eclipse.che.commons.schedule.executor.ThreadPullLauncher; + +/** Monitored and traced implementation of {@code ThreadPullLauncher}. */ +@Singleton +public class ObservableThreadPullLauncher extends ThreadPullLauncher { + + @Inject + public ObservableThreadPullLauncher( + ExecutorServiceWrapper wrapper, @Named("schedule.core_pool_size") Integer corePoolSize) { + super( + wrapper.wrap( + new CronThreadPoolExecutor( + corePoolSize, + new ThreadFactoryBuilder() + .setNameFormat("Annotated-scheduler-%d") + .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) + .setDaemon(false) + .build()), + CronThreadPoolExecutor.class.getName())); + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/TracedCronExecutorService.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/TracedCronExecutorService.java new file mode 100644 index 0000000000000000000000000000000000000000..5a48554be61c4eae03271983df902c2f4d9574f2 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/TracedCronExecutorService.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import io.opentracing.Tracer; +import io.opentracing.contrib.concurrent.TracedRunnable; +import io.opentracing.contrib.concurrent.TracedScheduledExecutorService; +import java.util.concurrent.Future; +import javax.inject.Inject; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; +import org.eclipse.che.commons.schedule.executor.CronExpression; + +/** + * Executor which propagates span from parent thread to submitted. Optionally it creates parent span + * if traceWithActiveSpanOnly = false. + */ +public class TracedCronExecutorService extends TracedScheduledExecutorService + implements CronExecutorService { + + private final CronExecutorService delegate; + + @Inject + public TracedCronExecutorService(CronExecutorService delegate, Tracer tracer) { + super(delegate, tracer); + this.delegate = delegate; + } + + @Override + public Future<?> schedule(Runnable task, CronExpression expression) { + return delegate.schedule( + tracer.activeSpan() == null ? task : new TracedRunnable(task, tracer), expression); + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/TracedExecutorServiceWrapper.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/TracedExecutorServiceWrapper.java new file mode 100644 index 0000000000000000000000000000000000000000..94e769210f9e5158b6cca30e6604ecca23bb98c3 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/TracedExecutorServiceWrapper.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import io.opentracing.Tracer; +import io.opentracing.contrib.concurrent.TracedExecutorService; +import io.opentracing.contrib.concurrent.TracedScheduledExecutorService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import javax.inject.Inject; +import javax.inject.Singleton; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; + +/** + * Implementation of {@code ExecutorServiceWrapper} that add all sort of tracing capabilities with + * help of traced implementation. + */ +@Singleton +public class TracedExecutorServiceWrapper implements ExecutorServiceWrapper { + + private final Tracer tracer; + + @Inject + public TracedExecutorServiceWrapper(Tracer tracer) { + this.tracer = tracer; + } + + @Override + public ExecutorService wrap(ExecutorService executor, String name, String... tags) { + return new TracedExecutorService(executor, tracer); + } + + @Override + public ScheduledExecutorService wrap( + ScheduledExecutorService executor, String name, String... tags) { + return new TracedScheduledExecutorService(executor, tracer); + } + + @Override + public CronExecutorService wrap(CronExecutorService executor, String name, String... tags) { + return new TracedCronExecutorService(executor, tracer); + } +} diff --git a/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/deploy/ExecutorWrapperModule.java b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/deploy/ExecutorWrapperModule.java new file mode 100644 index 0000000000000000000000000000000000000000..f85de327a72e8b63f47e68ee7c9c397d7a8edd2b --- /dev/null +++ b/core/commons/che-core-commons-observability/src/main/java/org/eclipse/che/commons/observability/deploy/ExecutorWrapperModule.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability.deploy; + +import com.google.inject.AbstractModule; +import org.eclipse.che.commons.observability.*; +import org.eclipse.che.commons.schedule.executor.ThreadPullLauncher; + +public class ExecutorWrapperModule extends AbstractModule { + + @Override + protected void configure() { + + if (Boolean.parseBoolean(System.getenv("CHE_METRICS_ENABLED"))) { + if (Boolean.parseBoolean(System.getenv("CHE_TRACING_ENABLED"))) { + bind(ExecutorServiceWrapper.class) + .to(MeteredAndTracedExecutorServiceWrapper.class) + .asEagerSingleton(); + } else { + bind(ExecutorServiceWrapper.class) + .to(MeteredExecutorServiceWrapper.class) + .asEagerSingleton(); + } + } else { + if (Boolean.parseBoolean(System.getenv("CHE_TRACING_ENABLED"))) { + bind(ExecutorServiceWrapper.class) + .to(TracedExecutorServiceWrapper.class) + .asEagerSingleton(); + } else { + bind(ExecutorServiceWrapper.class).to(NoopExecutorServiceWrapper.class).asEagerSingleton(); + } + } + + bind(ThreadPullLauncher.class).to(ObservableThreadPullLauncher.class); + } +} diff --git a/core/commons/che-core-commons-observability/src/test/java/io/micrometer/core/instrument/internal/TimedCronExecutorServiceTest.java b/core/commons/che-core-commons-observability/src/test/java/io/micrometer/core/instrument/internal/TimedCronExecutorServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..5c09c0338a9ec870029c02bfd42adba0aee80170 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/test/java/io/micrometer/core/instrument/internal/TimedCronExecutorServiceTest.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package io.micrometer.core.instrument.internal; + +import static org.testng.Assert.assertEquals; + +import io.micrometer.core.instrument.MockClock; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.simple.SimpleConfig; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.text.ParseException; +import java.util.concurrent.CountDownLatch; +import org.eclipse.che.commons.schedule.executor.CronExpression; +import org.eclipse.che.commons.schedule.executor.CronThreadPoolExecutor; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class TimedCronExecutorServiceTest { + + private SimpleMeterRegistry registry; + + private Iterable<Tag> userTags = Tags.of("userTagKey", "userTagValue"); + + @BeforeMethod + public void setup() { + registry = new SimpleMeterRegistry(SimpleConfig.DEFAULT, new MockClock()); + } + + @Test + public void executor() throws InterruptedException, ParseException { + // given + TimedCronExecutorService executorService = + new TimedCronExecutorService( + registry, + new CronThreadPoolExecutor(1), + TimedCronExecutorServiceTest.class.getName(), + userTags); + CountDownLatch lock = new CountDownLatch(1); + // when + executorService.schedule( + () -> { + lock.countDown(); + }, + // one time per second + new CronExpression(" * * * ? * * *")); + + lock.await(); + // then + assertEquals( + registry + .get("executor.scheduled.cron") + .tags(userTags) + .tag("name", TimedCronExecutorServiceTest.class.getName()) + .counter() + .count(), + 1.0); + assertEquals( + registry + .get("executor") + .tags(userTags) + .tag("name", TimedCronExecutorServiceTest.class.getName()) + .timer() + .count(), + 1L); + } +} diff --git a/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/CountedRejectedExecutionHandlerTest.java b/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/CountedRejectedExecutionHandlerTest.java new file mode 100644 index 0000000000000000000000000000000000000000..0120a1262b4f5a2c023f1419b397c8bbcd2c4708 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/CountedRejectedExecutionHandlerTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import static org.testng.Assert.assertEquals; + +import io.micrometer.core.instrument.MockClock; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.simple.SimpleConfig; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class CountedRejectedExecutionHandlerTest { + + private SimpleMeterRegistry registry; + private Iterable<Tag> userTags = Tags.of("userTagKey", "userTagValue"); + + @BeforeMethod + public void setup() { + registry = new SimpleMeterRegistry(SimpleConfig.DEFAULT, new MockClock()); + } + + @Test + public void countRejections() { + // given + ThreadPoolExecutor executor = + new ThreadPoolExecutor(0, 1, 60L, TimeUnit.SECONDS, new SynchronousQueue<>()); + executor.setRejectedExecutionHandler((r, executor1) -> {}); + + CountedRejectedExecutionHandler.monitorRejections( + registry, executor, CountedRejectedExecutionHandler.class.getName(), userTags); + CountDownLatch runnableTaskComplete = new CountDownLatch(1); + Runnable stub = + () -> { + try { + runnableTaskComplete.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new IllegalStateException("runnable interrupted before completion"); + } + }; + executor.submit(stub); + + // then + for (int i = 0; i < 14; i++) { + executor.submit( + () -> { + // do nothing. Task has to be rejected. + }); + } + // when + assertEquals( + registry + .get("executor.rejected") + .tags(userTags) + .tag("name", CountedRejectedExecutionHandler.class.getName()) + .counter() + .count(), + 14.0); + // cleanup + runnableTaskComplete.countDown(); + executor.shutdownNow(); + } +} diff --git a/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/CountedThreadFactoryTest.java b/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/CountedThreadFactoryTest.java new file mode 100644 index 0000000000000000000000000000000000000000..20f8c8aa39af5fa19a547e04e495052f0b07c559 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/CountedThreadFactoryTest.java @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; + +import io.micrometer.core.instrument.MockClock; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.simple.SimpleConfig; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class CountedThreadFactoryTest { + + private SimpleMeterRegistry registry; + private Iterable<Tag> userTags = Tags.of("userTagKey", "userTagValue"); + + @BeforeMethod + public void setup() { + registry = new SimpleMeterRegistry(SimpleConfig.DEFAULT, new MockClock()); + } + + @Test + public void shouldCountCreatedThreads() { + // given + ThreadFactory factory = + new CountedThreadFactory( + Executors.defaultThreadFactory(), + registry, + CountedThreadFactoryTest.class.getName(), + userTags); + // when + factory.newThread(() -> {}); + + // then + assertEquals( + registry + .get("thread.factory.created") + .tags(userTags) + .tag("name", CountedThreadFactoryTest.class.getName()) + .counter() + .count(), + 1.0); + } + + @Test + public void shouldCountRunningThreads() throws InterruptedException { + // given + ThreadFactory factory = + new CountedThreadFactory( + Executors.defaultThreadFactory(), + registry, + CountedThreadFactoryTest.class.getName(), + userTags); + + CountDownLatch runnableTaskStart = new CountDownLatch(1); + CountDownLatch runnableTaskComplete = new CountDownLatch(1); + Runnable task = + () -> { + runnableTaskStart.countDown(); + try { + Assert.assertTrue(runnableTaskComplete.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new IllegalStateException("runnable interrupted before completion"); + } + }; + + // when + Thread thread = factory.newThread(task); + thread.start(); + + // then + runnableTaskStart.await(); + assertEquals( + registry + .get("thread.factory.running") + .tags(userTags) + .tag("name", CountedThreadFactoryTest.class.getName()) + .gauge() + .value(), + 1.0); + + runnableTaskComplete.countDown(); + thread.join(); + assertEquals( + registry + .get("thread.factory.running") + .tags(userTags) + .tag("name", CountedThreadFactoryTest.class.getName()) + .gauge() + .value(), + 0.0); + + // put here to ensure that thread are not GCd + assertFalse(thread.isAlive()); + // put here to ensure that factory are not GCd + assertNotNull(factory); + } + + @Test + public void shouldCountRunningAndTerminatedThreadsInExecutorPool() + throws InterruptedException, TimeoutException, ExecutionException { + // given + JoinableThreadFactory factory = + new JoinableThreadFactory( + new CountedThreadFactory( + Executors.defaultThreadFactory(), + registry, + CountedThreadFactoryTest.class.getName(), + userTags)); + ExecutorService executor = Executors.newCachedThreadPool(factory); + + CountDownLatch runnableTaskStart = new CountDownLatch(10); + CountDownLatch runnableTaskComplete = new CountDownLatch(1); + + Runnable task = + () -> { + runnableTaskStart.countDown(); + try { + Assert.assertTrue(runnableTaskComplete.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new IllegalStateException("runnable interrupted before completion"); + } + }; + List<Future> futures = new ArrayList<>(10); + for (int i = 0; i < 10; i++) { + futures.add(executor.submit(task)); + } + runnableTaskStart.await(); + assertEquals( + registry + .get("thread.factory.running") + .tags(userTags) + .tag("name", CountedThreadFactoryTest.class.getName()) + .gauge() + .value(), + 10.0); + assertEquals( + registry + .get("thread.factory.terminated") + .tags(userTags) + .tag("name", CountedThreadFactoryTest.class.getName()) + .counter() + .count(), + 0.0); + + runnableTaskComplete.countDown(); + + for (Future future : futures) { + future.get(1, TimeUnit.MINUTES); + } + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + factory.joinAll(); + assertEquals( + registry + .get("thread.factory.running") + .tags(userTags) + .tag("name", CountedThreadFactoryTest.class.getName()) + .gauge() + .value(), + 0.0); + assertEquals( + registry + .get("thread.factory.terminated") + .tags(userTags) + .tag("name", CountedThreadFactoryTest.class.getName()) + .counter() + .count(), + 10.0); + // put here to ensure that factory are not GCd + assertNotNull(factory); + } + + public static class JoinableThreadFactory implements ThreadFactory { + + private final ThreadFactory delegate; + private final List<Thread> threads; + + public JoinableThreadFactory(ThreadFactory delegate) { + this.delegate = delegate; + this.threads = new ArrayList<>(); + } + + @Override + public Thread newThread(Runnable r) { + Thread result = delegate.newThread(r); + threads.add(result); + return result; + } + + public void joinAll() throws InterruptedException { + for (Thread thread : threads) { + if (thread.isAlive()) { + thread.join(); + } + } + } + } +} diff --git a/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/MeteredExecutorServiceWrapperTest.java b/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/MeteredExecutorServiceWrapperTest.java new file mode 100644 index 0000000000000000000000000000000000000000..57bdaf24218456b25bd53aa4b1c593a4b8a6dd32 --- /dev/null +++ b/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/MeteredExecutorServiceWrapperTest.java @@ -0,0 +1,459 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.MockClock; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.simple.SimpleConfig; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.text.ParseException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; +import org.eclipse.che.commons.schedule.executor.CronExpression; +import org.eclipse.che.commons.schedule.executor.CronThreadPoolExecutor; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class MeteredExecutorServiceWrapperTest { + + private MeterRegistry registry; + + private ExecutorServiceWrapper executorServiceWrapper; + private Iterable<Tag> userTags = Tags.of("userTagKey", "userTagValue"); + + private ExecutorService executor; + + @BeforeMethod + public void setup() { + registry = new SimpleMeterRegistry(SimpleConfig.DEFAULT, new MockClock()); + executorServiceWrapper = new MeteredExecutorServiceWrapper(registry); + } + + @AfterMethod + public void cleanup() { + if (executor == null) return; + // Tell threads to finish off. + executor.shutdown(); // Disable new tasks from being submitted + try { + // Wait a while for existing tasks to terminate + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) { + executor.shutdownNow(); // Cancel currently executing tasks + // Wait a while for tasks to respond to being cancelled + if (!executor.awaitTermination(60, TimeUnit.SECONDS)) + System.out.println("Pool did not terminate"); + } + } catch (InterruptedException ie) { + // (Re-)Cancel if current thread also interrupted + executor.shutdownNow(); + // Preserve interrupt status + Thread.currentThread().interrupt(); + } + } + + @Test + public void shouldRecordExecutorServiceMetrics() + throws InterruptedException, TimeoutException, ExecutionException { + // given + executor = + executorServiceWrapper.wrap( + Executors.newSingleThreadExecutor(), + MeteredExecutorServiceWrapperTest.class.getName(), + "userTagKey", + "userTagValue"); + CountDownLatch runnableTaskStart = new CountDownLatch(1); + // when + Future<?> future = + executor.submit( + () -> { + runnableTaskStart.countDown(); + }); + // then + runnableTaskStart.await(10, TimeUnit.SECONDS); + future.get(1, TimeUnit.MINUTES); + + assertEquals( + registry + .get("thread.factory.terminated") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 0.0); + assertEquals( + registry + .get("thread.factory.running") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .gauge() + .value(), + 1.0); + assertEquals( + registry + .get("thread.factory.created") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 1.0); + assertEquals( + registry + .get("executor.rejected") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 0.0); + + assertEquals( + registry + .get("executor.pool.size") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 1.0); + assertEquals( + registry + .get("executor.completed") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .functionCounter() + .count(), + 1.0); + assertEquals( + registry + .get("executor.queued") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 0.0); + assertEquals( + registry + .get("executor.idle") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .timer() + .count(), + 1); + assertEquals( + registry + .get("executor.queue.remaining") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + new Double(Integer.MAX_VALUE)); + assertEquals( + registry + .get("executor") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .timer() + .count(), + 1); + assertEquals( + registry + .get("executor.active") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 0.0); + } + + @Test + public void shouldRecordScheduledExecutorServiceMetrics() throws InterruptedException { + // given + executor = + executorServiceWrapper.wrap( + Executors.newSingleThreadScheduledExecutor(), + MeteredExecutorServiceWrapperTest.class.getName(), + "userTagKey", + "userTagValue"); + CountDownLatch runnableTaskStart = new CountDownLatch(1); + // when + ((ScheduledExecutorService) executor) + .scheduleAtFixedRate( + () -> { + runnableTaskStart.countDown(); + }, + 0, + 100, + TimeUnit.SECONDS); + // then + + runnableTaskStart.await(10, TimeUnit.SECONDS); + + assertEquals( + registry + .get("thread.factory.terminated") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 0.0); + assertEquals( + registry + .get("thread.factory.running") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .gauge() + .value(), + 1.0); + assertEquals( + registry + .get("thread.factory.created") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 1.0); + assertEquals( + registry + .get("executor.rejected") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 0.0); + + assertEquals( + registry + .get("executor.pool.size") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 1.0); + assertWithRetry( + () -> + registry + .get("executor.completed") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .functionCounter() + .count(), + 1.0, + 10, + 50); + + assertEquals( + registry + .get("executor.queued") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 1.0); + assertEquals( + registry + .get("executor.idle") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .timer() + .count(), + 0); + assertEquals( + registry + .get("executor.queue.remaining") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + new Double(Integer.MAX_VALUE)); + assertEquals( + registry + .get("executor") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .timer() + .count(), + 1); + assertEquals( + registry + .get("executor.active") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 0.0); + assertEquals( + registry + .get("executor.scheduled.once") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .counter() + .count(), + 0.0); + assertEquals( + registry + .get("executor.scheduled.repetitively") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .counter() + .count(), + 1.0); + } + + @Test + public void shouldRecordCronExecutorServiceMetrics() throws InterruptedException, ParseException { + // given + CronExecutorService executor = + executorServiceWrapper.wrap( + new CronThreadPoolExecutor(1), + MeteredExecutorServiceWrapperTest.class.getName(), + "userTagKey", + "userTagValue"); + CountDownLatch runnableTaskStart = new CountDownLatch(1); + // when + executor.schedule( + () -> { + runnableTaskStart.countDown(); + }, + new CronExpression(" * * * ? * * *")); + // then + runnableTaskStart.await(10, TimeUnit.SECONDS); + assertEquals( + registry + .get("thread.factory.terminated") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 0.0); + assertEquals( + registry + .get("thread.factory.running") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .gauge() + .value(), + 2.0); + assertEquals( + registry + .get("thread.factory.created") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 2.0); + assertEquals( + registry + .get("executor.rejected") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .counter() + .count(), + 0.0); + + assertEquals( + registry + .get("executor.pool.size") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 2.0); + assertEquals( + registry + .get("executor.completed") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .functionCounter() + .count(), + 1.0); + assertWithRetry( + () -> + registry + .get("executor.queued") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 1.0, + 10, + 50); + assertEquals( + registry + .get("executor.idle") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .timer() + .count(), + 0); + assertTrue( + registry + .get("executor.queue.remaining") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value() + > 0.0); + assertEquals( + registry + .get("executor") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .timer() + .count(), + 1); + assertEquals( + registry + .get("executor.active") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .gauge() + .value(), + 1.0); + assertEquals( + registry + .get("executor.scheduled.once") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .counter() + .count(), + 0.0); + assertEquals( + registry + .get("executor.scheduled.repetitively") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .counter() + .count(), + 0.0); + assertEquals( + registry + .get("executor.scheduled.cron") + .tag("name", MeteredExecutorServiceWrapperTest.class.getName()) + .tags(userTags) + .counter() + .count(), + 1.0); + } + + public <V> void assertWithRetry(Supplier<V> predicate, V expected, int times, int pause_millis) + throws InterruptedException { + for (int i = 0; i <= times; i++) { + V actual = predicate.get(); + if (expected.equals(actual)) { + return; + } else if (i + 1 <= times) { + Thread.sleep(pause_millis); + } + } + Assert.fail("Not able to get expected value " + expected + " with " + times + " retries"); + } +} diff --git a/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/NoopExecutorServiceWrapperTest.java b/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/NoopExecutorServiceWrapperTest.java new file mode 100644 index 0000000000000000000000000000000000000000..33a3db843485e1a4abfa0cb9979e8001eec41f1b --- /dev/null +++ b/core/commons/che-core-commons-observability/src/test/java/org/eclipse/che/commons/observability/NoopExecutorServiceWrapperTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.commons.observability; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.eclipse.che.commons.schedule.executor.CronExecutorService; +import org.eclipse.che.commons.schedule.executor.CronThreadPoolExecutor; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class NoopExecutorServiceWrapperTest { + NoopExecutorServiceWrapper noopExecutorServiceWrapper = new NoopExecutorServiceWrapper(); + + @Test + public void testWrapExecutorService() { + // given + ExecutorService executorService = Executors.newSingleThreadExecutor(); + // when + ExecutorService result = + noopExecutorServiceWrapper.wrap( + executorService, NoopExecutorServiceWrapper.class.getName(), "key", "value"); + // then + Assert.assertSame(result, executorService); + } + + @Test + public void testWrapScheduledExecutorService() { + // given + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + // when + ScheduledExecutorService result = + noopExecutorServiceWrapper.wrap( + executorService, NoopExecutorServiceWrapper.class.getName(), "key", "value"); + // then + Assert.assertSame(result, executorService); + } + + @Test + public void testWrapCronExecutorService() { + // given + CronExecutorService executorService = new CronThreadPoolExecutor(1); + // when + CronExecutorService result = + noopExecutorServiceWrapper.wrap( + executorService, NoopExecutorServiceWrapper.class.getName(), "key", "value"); + // then + Assert.assertSame(result, executorService); + } +} diff --git a/core/commons/che-core-commons-observability/src/test/resources/logback-test.xml b/core/commons/che-core-commons-observability/src/test/resources/logback-test.xml new file mode 100644 index 0000000000000000000000000000000000000000..e6abf05f024889739137c694dfc5d92712d86ced --- /dev/null +++ b/core/commons/che-core-commons-observability/src/test/resources/logback-test.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Copyright (c) 2012-2018 Red Hat, Inc. + This program and the accompanying materials are made + available under the terms of the Eclipse Public License 2.0 + which is available at https://www.eclipse.org/legal/epl-2.0/ + + SPDX-License-Identifier: EPL-2.0 + + Contributors: + Red Hat, Inc. - initial API and implementation + +--> +<configuration> + + <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%-41(%date[%.15thread]) %-45([%-5level] [%.30logger{30} %L]) - %msg%n</pattern> + </encoder> + </appender> + + <root level="INFO"> + <appender-ref ref="stdout"/> + </root> + +</configuration> diff --git a/core/commons/che-core-commons-schedule/src/main/java/org/eclipse/che/commons/schedule/executor/ThreadPullLauncher.java b/core/commons/che-core-commons-schedule/src/main/java/org/eclipse/che/commons/schedule/executor/ThreadPullLauncher.java index 8a96c9ec86f889e45747c70aa1770afc2cc04bdb..10e85f2ca8a0792bc6a575b88a097232ada35b2c 100644 --- a/core/commons/che-core-commons-schedule/src/main/java/org/eclipse/che/commons/schedule/executor/ThreadPullLauncher.java +++ b/core/commons/che-core-commons-schedule/src/main/java/org/eclipse/che/commons/schedule/executor/ThreadPullLauncher.java @@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory; @Singleton public class ThreadPullLauncher implements Launcher { private static final Logger LOG = LoggerFactory.getLogger(CronThreadPoolExecutor.class); - private final CronThreadPoolExecutor service; + private final CronExecutorService service; /** * @param corePoolSize the number of threads to keep in the pool, even if they are idle, unless @@ -41,14 +41,18 @@ public class ThreadPullLauncher implements Launcher { */ @Inject public ThreadPullLauncher(@Named("schedule.core_pool_size") Integer corePoolSize) { - this.service = + this( new CronThreadPoolExecutor( corePoolSize, new ThreadFactoryBuilder() .setNameFormat("Annotated-scheduler-%d") .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) .setDaemon(false) - .build()); + .build())); + } + + protected ThreadPullLauncher(CronExecutorService service) { + this.service = service; } @PreDestroy diff --git a/core/commons/pom.xml b/core/commons/pom.xml index 1d1983eeeaabddbadf664ac5e30d8ec8705aa917..4a67a9807aabca4cc1039d1ec29142fc148dd6b5 100644 --- a/core/commons/pom.xml +++ b/core/commons/pom.xml @@ -36,5 +36,6 @@ <module>che-core-commons-j2ee</module> <module>che-core-commons-mail</module> <module>che-core-commons-tracing</module> + <module>che-core-commons-observability</module> </modules> </project> diff --git a/deploy/openshift/templates/monitoring/grafana-dashboards.yaml b/deploy/openshift/templates/monitoring/grafana-dashboards.yaml index 2c6d40e1e342deeee55cc98549a57b95c17fecfb..69a724b0eb10e3628c0757596cef4158eaf96eda 100644 --- a/deploy/openshift/templates/monitoring/grafana-dashboards.yaml +++ b/deploy/openshift/templates/monitoring/grafana-dashboards.yaml @@ -3626,11 +3626,12 @@ data: "gnetId": null, "graphTooltip": 1, "id": 2, - "iteration": 1569330248249, + "iteration": 1570978807775, "links": [], "panels": [ { "collapsed": false, + "datasource": "$datasource", "gridPos": { "h": 1, "w": 24, @@ -3818,6 +3819,7 @@ data: }, { "collapsed": false, + "datasource": "$datasource", "gridPos": { "h": 1, "w": 24, @@ -4251,7 +4253,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(che_workspace_start_time_seconds_sum[1h])/rate(che_workspace_start_time_seconds_count [1h])", + "expr": "increase(che_workspace_start_time_seconds_sum[1h])/increase(che_workspace_start_time_seconds_count [1h])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4343,7 +4345,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(che_workspace_stop_time_seconds_sum[1h])/rate(che_workspace_stop_time_seconds_count [1h])", + "expr": "increase(che_workspace_stop_time_seconds_sum[1h])/increase(che_workspace_stop_time_seconds_count [1h])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4615,7 +4617,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(che_workspace_start_time_seconds_bucket{le=\"62.992853672\",result=\"success\"}[1h]) / ignoring(le) rate(che_workspace_start_time_seconds_count{result=\"success\"}[1h])", + "expr": "increase(che_workspace_start_time_seconds_bucket{le=\"62.992853672\",result=\"success\"}[1h]) / ignoring(le) increase(che_workspace_start_time_seconds_count{result=\"success\"}[1h])", "format": "time_series", "intervalFactor": 1, "legendFormat": "workspace start ratio", @@ -4759,6 +4761,7 @@ data: }, { "collapsed": false, + "datasource": "$datasource", "gridPos": { "h": 1, "w": 24, @@ -4864,6 +4867,7 @@ data: }, { "collapsed": false, + "datasource": "$datasource", "gridPos": { "h": 1, "w": 24, @@ -4968,6 +4972,7 @@ data: }, { "collapsed": false, + "datasource": "$datasource", "gridPos": { "h": 1, "w": 24, @@ -5498,6 +5503,7 @@ data: }, { "collapsed": false, + "datasource": "$datasource", "gridPos": { "h": 1, "w": 24, @@ -5506,7 +5512,7 @@ data: }, "id": 22, "panels": [], - "title": "JsonRPC", + "title": "Executors", "type": "row" }, { @@ -5519,17 +5525,19 @@ data: "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, + "w": 12, "x": 0, "y": 50 }, "id": 18, + "interval": "", "legend": { "alignAsTable": false, - "avg": false, + "avg": true, "current": true, "max": true, "min": false, + "rightSide": false, "show": true, "total": false, "values": true @@ -5548,47 +5556,23 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "executor_pool_size_threads{name=\"che.core.jsonrpc.major_executor\",}", + "expr": "thread_factory_running_threads", "format": "time_series", "hide": false, "intervalFactor": 1, - "legendFormat": "major size", + "legendFormat": "{{name}}", "refId": "A" - }, - { - "expr": "executor_active_threads{name=\"che.core.jsonrpc.major_executor\",}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "major active", - "refId": "B" - }, - { - "expr": "executor_pool_size_threads{name=\"che.core.jsonrpc.minor_executor\",}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "minor size", - "refId": "C" - }, - { - "expr": "executor_active_threads{name=\"che.core.jsonrpc.minor_executor\",}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "minor active", - "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "che.core.jsonrpc.executor_pool", + "title": "Thread running", "tooltip": { "shared": true, "sort": 0, @@ -5605,10 +5589,10 @@ data: "yaxes": [ { "format": "short", - "label": null, + "label": "", "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5617,7 +5601,7 @@ data: "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -5635,16 +5619,15 @@ data: "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 8, + "w": 12, + "x": 12, "y": 50 }, - "id": 26, + "id": 30, "legend": { - "alignAsTable": false, "avg": false, "current": true, - "max": true, + "max": false, "min": false, "show": true, "total": false, @@ -5668,27 +5651,18 @@ data: "steppedLine": false, "targets": [ { - "expr": "executor_queued_threads{name=\"che.core.jsonrpc.major_executor\",}", + "expr": "thread_factory_terminated_threads_total", "format": "time_series", - "instant": false, "intervalFactor": 1, - "legendFormat": "major", + "legendFormat": "{{name}}", "refId": "A" - }, - { - "expr": "executor_queued_threads{name=\"che.core.jsonrpc.minor_executor\",}", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "minor", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "JsonRpcQueue", + "title": "Thread terminated", "tooltip": { "shared": true, "sort": 0, @@ -5716,8 +5690,8 @@ data: "label": null, "logBase": 1, "max": null, - "min": "0", - "show": true + "min": null, + "show": false } ], "yaxis": { @@ -5736,9 +5710,9 @@ data: "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 50 + "w": 12, + "x": 0, + "y": 57 }, "id": 32, "legend": { @@ -5747,7 +5721,7 @@ data: "current": true, "hideEmpty": false, "hideZero": false, - "max": true, + "max": false, "min": false, "show": true, "total": false, @@ -5771,27 +5745,19 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(deriv(executor_rejected_tasks_total{name=\"che.core.jsonrpc.major_executor\",}[1m]))", + "expr": "thread_factory_created_threads_total", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "major", + "legendFormat": "{{name}}", "refId": "A" - }, - { - "expr": "sum(deriv(executor_rejected_tasks_total{name=\"che.core.jsonrpc.minor_executor\",}[1m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "minor", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Rejected message/minute", + "title": "Thread created", "tooltip": { "shared": true, "sort": 0, @@ -5821,7 +5787,7 @@ data: "logBase": 1, "max": null, "min": "0", - "show": true + "show": false } ], "yaxis": { @@ -5835,15 +5801,17 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 0, + "w": 12, + "x": 12, "y": 57 }, - "id": 30, + "id": 28, + "interval": "", "legend": { "avg": false, "current": true, @@ -5871,25 +5839,19 @@ data: "steppedLine": false, "targets": [ { - "expr": "executor_seconds_max{name=\"che.core.jsonrpc.major_executor\",}", + "expr": "increase(thread_factory_created_threads_total[1m])", "format": "time_series", + "interval": "", "intervalFactor": 1, - "legendFormat": "major", + "legendFormat": "{{name}}", "refId": "A" - }, - { - "expr": "executor_seconds_max{name=\"che.core.jsonrpc.minor_executor\",}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "minor", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Seconds Max", + "title": "Created thread/minute", "tooltip": { "shared": true, "sort": 0, @@ -5909,7 +5871,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true }, { @@ -5917,7 +5879,7 @@ data: "label": null, "logBase": 1, "max": null, - "min": null, + "min": "0", "show": true } ], @@ -5936,12 +5898,14 @@ data: "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 8, - "y": 57 + "w": 12, + "x": 0, + "y": 64 }, - "id": 34, + "id": 26, + "interval": "", "legend": { + "alignAsTable": false, "avg": false, "current": true, "max": true, @@ -5968,26 +5932,19 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(deriv(executor_seconds_sum{name=\"che.core.jsonrpc.major_executor\",}[1m]))", + "expr": "executor_active_threads", "format": "time_series", - "interval": "", + "instant": false, "intervalFactor": 1, - "legendFormat": "major", + "legendFormat": "{{name}}", "refId": "A" - }, - { - "expr": "sum(deriv(executor_seconds_sum{name=\"che.core.jsonrpc.minor_executor\",}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "minor", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Seconds Sum", + "title": "Executor threads active", "tooltip": { "shared": true, "sort": 0, @@ -6016,7 +5973,7 @@ data: "logBase": 1, "max": null, "min": "0", - "show": true + "show": false } ], "yaxis": { @@ -6035,11 +5992,11 @@ data: "fillGradient": 0, "gridPos": { "h": 7, - "w": 8, - "x": 16, - "y": 57 + "w": 12, + "x": 12, + "y": 64 }, - "id": 28, + "id": 34, "legend": { "avg": false, "current": true, @@ -6067,27 +6024,19 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(deriv(executor_completed_tasks_total{name=\"che.core.jsonrpc.major_executor\",}[1m]))", + "expr": "executor_pool_size_threads", "format": "time_series", "interval": "", "intervalFactor": 1, - "legendFormat": "major", + "legendFormat": "{{name}}", "refId": "A" - }, - { - "expr": "sum(deriv(executor_completed_tasks_total{name=\"che.core.jsonrpc.minor_executor\",}[1m]))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "minor", - "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Completed Task/minute", + "title": "Executor pool size", "tooltip": { "shared": true, "sort": 0, @@ -6116,7 +6065,7 @@ data: "logBase": 1, "max": null, "min": "0", - "show": true + "show": false } ], "yaxis": { @@ -6125,146 +6074,2293 @@ data: } }, { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 61, - "panels": [], - "title": "Traces", - "type": "row" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 71 }, + "id": 85, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 55, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"WorkspaceRuntimes#startAsync\"}", - "legendFormat": "{{operation}}", - "refId": "F" - }, - { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"SidecarToolingProvisioner#provision\"}", - "legendFormat": "{{operation}}", - "refId": "A" - }, - { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftEnvironmentProvisioner#provision\"}", - "legendFormat": "{{operation}}", - "refId": "B" - }, - { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"UniqueWorkspacePVCStrategy#prepare\"}", - "legendFormat": "{{operation}}", - "refId": "C" - }, - { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#startMachines\"}", - "legendFormat": "{{operation}}", - "refId": "D" - }, - { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"WaitMachinesStart\"}", - "legendFormat": "{{operation}}", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Workspace Start Max", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "expr": "executor_queued_tasks", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queued tasks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] }, + "yaxes": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 84, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100*(1-executor_queue_remaining_tasks/ (executor_queue_remaining_tasks + executor_queued_tasks ))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queue occupancy", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": 100, + "min": "0", + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 100, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 83, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "executor_rejected_tasks_total", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rejected task", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 82, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(executor_rejected_tasks_total[1m])", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rejected task/minute", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 86, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "executor_completed_tasks_total", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Completed tasks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 88, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(executor_completed_tasks_total[1m])", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Completed tasks/minute ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 92 + }, + "id": 87, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "executor_seconds_max", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Tasks execution seconds max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 92 + }, + "id": 90, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(executor_seconds_sum[1h])/increase(executor_seconds_count[1h])", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Tasks execution seconds avg", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 99 + }, + "id": 89, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "executor_idle_seconds_max", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Executor idle seconds max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 99 + }, + "id": 81, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(executor_idle_seconds_sum[1h])/increase(executor_idle_seconds_count[1h])", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Executor idle seconds avg", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 106 + }, + "id": 92, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "executor_scheduled_repetitively_total", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scheduled repetitively", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 106 + }, + "id": 93, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "executor_scheduled_once_total", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Executor scheduled once", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 113 + }, + "id": 91, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "executor_scheduled_cron_total", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Executor scheduled cron", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "$datasource", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 120 + }, + "id": 61, + "panels": [], + "title": "Traces", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 121 + }, + "id": 55, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"WorkspaceRuntimes#startAsync\"}", + "legendFormat": "{{operation}}", + "refId": "A" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#start\"}", + "legendFormat": "{{operation}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Workspace Start Max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 121 + }, + "id": 56, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"WorkspaceRuntimes#startAsync\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"WorkspaceRuntimes#startAsync\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "A" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#start\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#start\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Workspace Start Avg", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 128 + }, + "id": 57, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"WorkspaceRuntimes#stopAsync\"}", + "legendFormat": "{{operation}}", + "refId": "A" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#internalStop\"}", + "legendFormat": "{{operation}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Workspace Stop Max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 128 + }, + "id": 58, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"WorkspaceRuntimes#stopAsync\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"WorkspaceRuntimes#stopAsync\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "A" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#internalStop\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#internalStop\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Workspace Stop Avg", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 135 + }, + "id": 51, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"SidecarToolingProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "A" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftEnvironmentProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "B" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"PerWorkspacePVCStrategy#prepare\"}", + "legendFormat": "{{operation}}", + "refId": "C" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#startMachines\"}", + "legendFormat": "{{operation}}", + "refId": "D" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"WaitMachinesStart\"}", + "legendFormat": "{{operation}}", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OpenShiftInternalRuntime#start Max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 135 + }, + "id": 54, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"SidecarToolingProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"SidecarToolingProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "A" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftEnvironmentProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftEnvironmentProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "B" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"PerWorkspacePVCStrategy#prepare\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"PerWorkspacePVCStrategy#prepare\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "C" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#startMachines\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#startMachines\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "D" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"WaitMachinesStart\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"WaitMachinesStart\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OpenShiftInternalRuntime#start avg", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 142 + }, + "id": 94, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"PluginBrokerManager#getTooling\"}", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Plugin Brokering Execution Max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 142 + }, + "id": 97, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"PluginBrokerManager#getTooling\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"PluginBrokerManager#getTooling\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Plugin Brokering Execution Avg", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 149 + }, + "id": 96, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"LogsVolumeMachineProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "A" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"ServersConverter#provision\"}", + "legendFormat": "{{operation}}", + "refId": "B" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"EnvVarsConverter#provision\"}", + "legendFormat": "{{operation}}", + "refId": "C" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"RestartPolicyRewriter#provision\"}", + "legendFormat": "{{operation}}", + "refId": "D" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"RouteTlsProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "E" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"RamLimitRequestProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "F" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"PodTerminationGracePeriodProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "G" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"ImagePullSecretProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "H" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"ProxySettingsProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "I" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"ServiceAccountProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "J" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"VcsSshKeysProvisioner#provision\"}", + "legendFormat": "{{operation}}", + "refId": "K" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OpenShiftEnvironmentProvisioner#provision Max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 149 + }, + "id": 98, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"LogsVolumeMachineProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"LogsVolumeMachineProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "A" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"ServersConverter#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"ServersConverter#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "B" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"EnvVarsConverter#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"EnvVarsConverter#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "C" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"RestartPolicyRewriter#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"RestartPolicyRewriter#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "D" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"RouteTlsProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"RouteTlsProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "E" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"RamLimitRequestProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"RamLimitRequestProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "F" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"PodTerminationGracePeriodProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"PodTerminationGracePeriodProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "G" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"ImagePullSecretProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"ImagePullSecretProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "H" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"ProxySettingsProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"ProxySettingsProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "I" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"ServiceAccountProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"ServiceAccountProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "J" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"VcsSshKeysProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"VcsSshKeysProvisioner#provision\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "K" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OpenShiftEnvironmentProvisioner#provision avg", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 65 + "x": 0, + "y": 156 }, - "id": 56, + "id": 100, "legend": { "alignAsTable": false, "avg": false, @@ -6278,7 +8374,7 @@ data: }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "null as zero", "options": { "dataLinks": [] }, @@ -6288,45 +8384,40 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"WorkspaceRuntimes#startAsync\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"WorkspaceRuntimes#startAsync\"}[1h])", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftEnvironmentProvisioner#provision\"}", "legendFormat": "{{operation}}", "refId": "A" }, { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"SidecarToolingProvisioner#provision\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"SidecarToolingProvisioner#provision\"}[1h])", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"PrepareStorage\"}", "legendFormat": "{{operation}}", "refId": "B" }, { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftEnvironmentProvisioner#provision\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftEnvironmentProvisioner#provision\"}[1h])", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"PerWorkspacePVCStrategy#prepare\"}", "legendFormat": "{{operation}}", "refId": "C" }, { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"UniqueWorkspacePVCStrategy#prepare\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"UniqueWorkspacePVCStrategy#prepare\"}[1h])", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"DeployBroker\"}", "legendFormat": "{{operation}}", "refId": "D" }, { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#startMachines\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#startMachines\"}[1h])", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"WaitBrokerResult\"}", "legendFormat": "{{operation}}", "refId": "E" - }, - { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"WaitMachinesStart\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"WaitMachinesStart\"}[1h])", - "legendFormat": "{{operation}}", - "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Workspace Start Avg", + "title": "Plugin Brokering Execution Max", "tooltip": { "shared": true, "sort": 0, @@ -6355,7 +8446,7 @@ data: "logBase": 1, "max": null, "min": 0, - "show": true + "show": false } ], "yaxis": { @@ -6369,15 +8460,16 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, - "x": 0, - "y": 72 + "x": 12, + "y": 156 }, - "id": 51, + "id": 99, "legend": { "alignAsTable": false, "avg": false, @@ -6401,45 +8493,40 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#createSecrets\"}", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftEnvironmentProvisioner#provision\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftEnvironmentProvisioner#provision\"}[1h])", "legendFormat": "{{operation}}", "refId": "A" }, { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#createConfigMaps\"}", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"PrepareStorage\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"PPrepareStorage\"}[1h])", "legendFormat": "{{operation}}", "refId": "B" }, { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#createServices\"}", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"PerWorkspacePVCStrategy#prepare\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"PerWorkspacePVCStrategy#prepare\"}[1h])", "legendFormat": "{{operation}}", "refId": "C" }, { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#createRoutes\"}", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"DeployBroker\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"DeployBroker\"}[1h])", "legendFormat": "{{operation}}", "refId": "D" }, { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#doStartMachine\"}", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"WaitBrokerResult\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"WaitBrokerResult\"}[1h])", "legendFormat": "{{operation}}", "refId": "E" - }, - { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#listenEvents\"}", - "legendFormat": "{{operation}}", - "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "OpenShiftInternalRuntime#startMachines Max", + "title": "Plugin Brokering Execution Avg", "tooltip": { "shared": true, "sort": 0, @@ -6468,7 +8555,7 @@ data: "logBase": 1, "max": null, "min": 0, - "show": true + "show": false } ], "yaxis": { @@ -6482,15 +8569,16 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, - "x": 12, - "y": 72 + "x": 0, + "y": 163 }, - "id": 54, + "id": 95, "legend": { "alignAsTable": false, "avg": false, @@ -6514,45 +8602,119 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#createSecrets\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#createSecrets\"}[1h])", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"WaitRunningAsync\"}", "legendFormat": "{{operation}}", "refId": "A" }, { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#createConfigMaps\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#createConfigMaps\"}[1h])", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"CheckServers\"}", "legendFormat": "{{operation}}", "refId": "B" - }, + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "WaitMachinesStart Max", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#createServices\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#createServices\"}[1h])", - "legendFormat": "{{operation}}", - "refId": "C" + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true }, { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#createRoutes\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#createRoutes\"}[1h])", - "legendFormat": "{{operation}}", - "refId": "D" - }, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 163 + }, + "id": 101, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#listenEvents\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#listenEvents\"}[1h])", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"WaitRunningAsync\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"WaitRunningAsync\"}[1h])", "legendFormat": "{{operation}}", - "refId": "E" + "refId": "A" }, { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#doStartMachine\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#doStartMachine\"}[1h])", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"CheckServers\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"CheckServers\"}[1h])", "legendFormat": "{{operation}}", - "refId": "F" + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "OpenShiftInternalRuntime#startMachines Avg", + "title": "WaitMachinesStart avg", "tooltip": { "shared": true, "sort": 0, @@ -6581,7 +8743,7 @@ data: "logBase": 1, "max": null, "min": 0, - "show": true + "show": false } ], "yaxis": { @@ -6595,15 +8757,16 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, - "y": 79 + "y": 170 }, - "id": 57, + "id": 102, "legend": { "alignAsTable": false, "avg": false, @@ -6627,25 +8790,45 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"WorkspaceRuntimes#stopAsync\"}", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#createSecrets\"}", "legendFormat": "{{operation}}", "refId": "A" }, { - "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#internalStop\"}", + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#createConfigMaps\"}", "legendFormat": "{{operation}}", "refId": "B" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#createServices\"}", + "legendFormat": "{{operation}}", + "refId": "C" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#createRoutes\"}", + "legendFormat": "{{operation}}", + "refId": "D" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#listenEvents\"}", + "legendFormat": "{{operation}}", + "refId": "E" + }, + { + "expr": "che_server_api_tracing_span_seconds_max{operation=\"OpenShiftInternalRuntime#doStartMachine\"}", + "legendFormat": "{{operation}}", + "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Workspace Stop Max", + "title": "OpenShiftInternalRuntime#startMachines Max", "tooltip": { "shared": true, "sort": 0, @@ -6674,7 +8857,7 @@ data: "logBase": 1, "max": null, "min": 0, - "show": true + "show": false } ], "yaxis": { @@ -6688,15 +8871,16 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", + "description": "", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, - "y": 79 + "y": 170 }, - "id": 58, + "id": 103, "legend": { "alignAsTable": false, "avg": false, @@ -6720,25 +8904,45 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"WorkspaceRuntimes#stopAsync\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"WorkspaceRuntimes#stopAsync\"}[1h])", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#createSecrets\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#createSecrets\"}[1h])", "legendFormat": "{{operation}}", "refId": "A" }, { - "expr": "rate(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#internalStop\"}[1h]) / rate(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#internalStop\"}[1h])", + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#createConfigMaps\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#createConfigMaps\"}[1h])", "legendFormat": "{{operation}}", "refId": "B" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#createServices\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#createServices\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "C" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#createRoutes\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#createRoutes\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "D" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#listenEvents\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#listenEvents\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "E" + }, + { + "expr": "increase(che_server_api_tracing_span_seconds_sum{operation=\"OpenShiftInternalRuntime#doStartMachine\"}[1h]) / increase(che_server_api_tracing_span_seconds_count{operation=\"OpenShiftInternalRuntime#doStartMachine\"}[1h])", + "legendFormat": "{{operation}}", + "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Workspace Stop Avg", + "title": "OpenShiftInternalRuntime#startMachines avg", "tooltip": { "shared": true, "sort": 0, @@ -6767,7 +8971,7 @@ data: "logBase": 1, "max": null, "min": 0, - "show": true + "show": false } ], "yaxis": { @@ -6777,11 +8981,12 @@ data: }, { "collapsed": true, + "datasource": "$datasource", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 86 + "y": 177 }, "id": 42, "panels": [ diff --git a/infrastructures/kubernetes/pom.xml b/infrastructures/kubernetes/pom.xml index df0c5e5bdcc90c68c2cbbd9944e6d844cc1862d8..6ace70980335eff3dafeeeb993ab137f57f93fad 100644 --- a/infrastructures/kubernetes/pom.xml +++ b/infrastructures/kubernetes/pom.xml @@ -150,6 +150,10 @@ <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-lang</artifactId> </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-observability</artifactId> + </dependency> <dependency> <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-schedule</artifactId> diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java index 9a583917cc06ce10fb9d373c36ca4a15af2a2227..5c9e537713c4a652c5e88ff3437892398cb3ed05 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelper.java @@ -36,6 +36,7 @@ import javax.inject.Singleton; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; import org.eclipse.che.commons.lang.concurrent.ThreadLocalPropagateContext; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesDeployments; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; import org.eclipse.che.workspace.infrastructure.kubernetes.provision.SecurityContextProvisioner; @@ -85,19 +86,22 @@ public class PVCSubPathHelper { @Named("che.infra.kubernetes.pvc.jobs.memorylimit") String jobMemoryLimit, @Named("che.infra.kubernetes.pvc.jobs.image") String jobImage, KubernetesNamespaceFactory factory, - SecurityContextProvisioner securityContextProvisioner) { + SecurityContextProvisioner securityContextProvisioner, + ExecutorServiceWrapper executorServiceWrapper) { this.jobMemoryLimit = jobMemoryLimit; this.jobImage = jobImage; this.factory = factory; this.securityContextProvisioner = securityContextProvisioner; this.executor = - Executors.newFixedThreadPool( - COUNT_THREADS, - new ThreadFactoryBuilder() - .setNameFormat("PVCSubPathHelper-ThreadPool-%d") - .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) - .setDaemon(false) - .build()); + executorServiceWrapper.wrap( + Executors.newFixedThreadPool( + COUNT_THREADS, + new ThreadFactoryBuilder() + .setNameFormat("PVCSubPathHelper-ThreadPool-%d") + .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) + .setDaemon(false) + .build()), + PVCSubPathHelper.class.getName()); } /** diff --git a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSharedPool.java b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSharedPool.java index b9807822efeb81f2a0b2ce9220ba6199dde92085..3b4a7a38d6a547a7bb3cbc69ab1b49bf2c8ad683 100644 --- a/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSharedPool.java +++ b/infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/util/KubernetesSharedPool.java @@ -17,6 +17,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import javax.inject.Inject; import javax.inject.Singleton; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; /** * Provides single {@link ExecutorService} instance with daemon threads for Kubernetes/Openshfit @@ -30,13 +31,15 @@ public class KubernetesSharedPool { private final ExecutorService executor; @Inject - public KubernetesSharedPool() { + public KubernetesSharedPool(ExecutorServiceWrapper executorServiceWrapper) { final ThreadFactory factory = new ThreadFactoryBuilder() .setNameFormat("KubernetesMachineSharedPool-%d") .setDaemon(true) .build(); - this.executor = Executors.newCachedThreadPool(factory); + this.executor = + executorServiceWrapper.wrap( + Executors.newCachedThreadPool(factory), KubernetesSharedPool.class.getName()); } public ExecutorService getExecutor() { diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java index fe41dd62bc14a235a561bdc17638b5a37a2da60a..50b0e9d3b75947cbbdf25aa40d08efc10091f994 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntimeTest.java @@ -105,6 +105,7 @@ import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfi import org.eclipse.che.api.workspace.server.spi.provision.InternalEnvironmentProvisioner; import org.eclipse.che.api.workspace.shared.dto.event.MachineStatusEvent; import org.eclipse.che.api.workspace.shared.dto.event.RuntimeLogEvent; +import org.eclipse.che.commons.observability.NoopExecutorServiceWrapper; import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache; import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesRuntimeStateCache; import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment; @@ -250,7 +251,7 @@ public class KubernetesInternalRuntimeTest { probesScheduler, workspaceProbesFactory, eventPublisher, - new KubernetesSharedPool(), + new KubernetesSharedPool(new NoopExecutorServiceWrapper()), runtimeStatesCache, machinesCache, startSynchronizerFactory, diff --git a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java index 2da423520b71e4891f93d1889de5b631ced5168e..fa212098382df0af555e96652db6565009d433ba 100644 --- a/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java +++ b/infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PVCSubPathHelperTest.java @@ -37,6 +37,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Stream; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; +import org.eclipse.che.commons.observability.NoopExecutorServiceWrapper; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesDeployments; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace; import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory; @@ -79,7 +80,11 @@ public class PVCSubPathHelperTest { public void setup() throws Exception { pvcSubPathHelper = new PVCSubPathHelper( - jobMemoryLimit, jobImage, k8sNamespaceFactory, securityContextProvisioner); + jobMemoryLimit, + jobImage, + k8sNamespaceFactory, + securityContextProvisioner, + new NoopExecutorServiceWrapper()); lenient().when(k8sNamespaceFactory.create(anyString())).thenReturn(k8sNamespace); lenient().when(k8sNamespace.deployments()).thenReturn(osDeployments); lenient().when(pod.getStatus()).thenReturn(podStatus); diff --git a/multiuser/integration-tests/che-multiuser-cascade-removal/pom.xml b/multiuser/integration-tests/che-multiuser-cascade-removal/pom.xml index 53f151f5821d8d7482523c84ece671339f589005..5bc3f61510dcad8d23bc25364110d86b90c09f58 100644 --- a/multiuser/integration-tests/che-multiuser-cascade-removal/pom.xml +++ b/multiuser/integration-tests/che-multiuser-cascade-removal/pom.xml @@ -118,6 +118,11 @@ <artifactId>che-core-commons-lang</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-observability</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-test</artifactId> diff --git a/multiuser/integration-tests/che-multiuser-cascade-removal/src/test/java/org/eclipse/che/multiuser/integration/jpa/cascaderemoval/JpaEntitiesCascadeRemovalTest.java b/multiuser/integration-tests/che-multiuser-cascade-removal/src/test/java/org/eclipse/che/multiuser/integration/jpa/cascaderemoval/JpaEntitiesCascadeRemovalTest.java index f5e520cd1052d01ddb1bc9ede9b11a82213c4545..ec2a34cde7d7c22009222a78da134b312da7975b 100644 --- a/multiuser/integration-tests/che-multiuser-cascade-removal/src/test/java/org/eclipse/che/multiuser/integration/jpa/cascaderemoval/JpaEntitiesCascadeRemovalTest.java +++ b/multiuser/integration-tests/che-multiuser-cascade-removal/src/test/java/org/eclipse/che/multiuser/integration/jpa/cascaderemoval/JpaEntitiesCascadeRemovalTest.java @@ -81,6 +81,8 @@ import org.eclipse.che.api.workspace.server.spi.WorkspaceDao; import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironmentFactory; import org.eclipse.che.api.workspace.server.wsplugins.ChePluginsApplier; import org.eclipse.che.commons.env.EnvironmentContext; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; +import org.eclipse.che.commons.observability.NoopExecutorServiceWrapper; import org.eclipse.che.commons.subject.SubjectImpl; import org.eclipse.che.commons.test.db.H2DBTestServer; import org.eclipse.che.commons.test.db.H2JpaCleaner; @@ -218,6 +220,7 @@ public class JpaEntitiesCascadeRemovalTest { install(new MultiuserWorkspaceJpaModule()); install(new MachineAuthModule()); install(new DevfileModule()); + bind(ExecutorServiceWrapper.class).to(NoopExecutorServiceWrapper.class); bind(FreeResourcesLimitDao.class).to(JpaFreeResourcesLimitDao.class); bind(RemoveFreeResourcesLimitSubscriber.class).asEagerSingleton(); @@ -236,7 +239,9 @@ public class JpaEntitiesCascadeRemovalTest { .annotatedWith(Names.named("che.workspace.auto_restore")) .toInstance(false); bind(WorkspaceSharedPool.class) - .toInstance(new WorkspaceSharedPool("cached", null, null, null)); + .toInstance( + new WorkspaceSharedPool( + "cached", null, null, new NoopExecutorServiceWrapper())); bind(String[].class) .annotatedWith(Names.named("che.auth.reserved_user_names")) diff --git a/pom.xml b/pom.xml index c013f3de10507ce1e164bc589a0bfc5567a0edbc..304293f111372a42be26c1e6aba97423f0016034 100644 --- a/pom.xml +++ b/pom.xml @@ -397,6 +397,11 @@ <artifactId>che-core-commons-mail</artifactId> <version>${che.version}</version> </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-observability</artifactId> + <version>${che.version}</version> + </dependency> <dependency> <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-schedule</artifactId> diff --git a/wsmaster/che-core-api-workspace/pom.xml b/wsmaster/che-core-api-workspace/pom.xml index 718d9b26043bad8eb1840967f1a067d2e8c48465..de2ad8890831bc17c901b0da406c1d3dcd079351 100644 --- a/wsmaster/che-core-api-workspace/pom.xml +++ b/wsmaster/che-core-api-workspace/pom.xml @@ -58,14 +58,6 @@ <groupId>com.google.inject.extensions</groupId> <artifactId>guice-assistedinject</artifactId> </dependency> - <dependency> - <groupId>io.opentracing</groupId> - <artifactId>opentracing-api</artifactId> - </dependency> - <dependency> - <groupId>io.opentracing.contrib</groupId> - <artifactId>opentracing-concurrent</artifactId> - </dependency> <dependency> <groupId>io.swagger</groupId> <artifactId>swagger-annotations</artifactId> @@ -126,6 +118,10 @@ <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-lang</artifactId> </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-observability</artifactId> + </dependency> <dependency> <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-schedule</artifactId> diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java index a5b5114762aebe944a6f590954967f2ff2ce1609..dbf5213e65984bcfdb0f758ba0d76051afeeb349 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/WorkspaceSharedPool.java @@ -14,8 +14,6 @@ package org.eclipse.che.api.workspace.server; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; -import io.opentracing.Tracer; -import io.opentracing.contrib.concurrent.TracedExecutorService; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -28,6 +26,7 @@ import javax.inject.Singleton; import org.eclipse.che.commons.annotation.Nullable; import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; import org.eclipse.che.commons.lang.concurrent.ThreadLocalPropagateContext; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +45,7 @@ public class WorkspaceSharedPool { @Named("che.workspace.pool.type") String poolType, @Named("che.workspace.pool.exact_size") @Nullable String exactSizeProp, @Named("che.workspace.pool.cores_multiplier") @Nullable String coresMultiplierProp, - Tracer tracer) { + ExecutorServiceWrapper wrapper) { ThreadFactory factory = new ThreadFactoryBuilder() @@ -56,7 +55,9 @@ public class WorkspaceSharedPool { .build(); switch (poolType.toLowerCase()) { case "cached": - executor = new TracedExecutorService(Executors.newCachedThreadPool(factory), tracer); + executor = + wrapper.wrap( + Executors.newCachedThreadPool(factory), WorkspaceSharedPool.class.getName()); break; case "fixed": Integer exactSize = exactSizeProp == null ? null : Ints.tryParse(exactSizeProp); @@ -71,7 +72,9 @@ public class WorkspaceSharedPool { size *= coresMultiplier; } } - executor = new TracedExecutorService(Executors.newFixedThreadPool(size, factory), tracer); + executor = + wrapper.wrap( + Executors.newFixedThreadPool(size, factory), WorkspaceSharedPool.class.getName()); break; default: throw new IllegalArgumentException( diff --git a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/ProbeScheduler.java b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/ProbeScheduler.java index 1d452b56e7a45bfecc9bbcb487f9e7e512158207..03e5aa4eff388a48f2342a79e8ed57a31ee7522f 100644 --- a/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/ProbeScheduler.java +++ b/wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/hc/probe/ProbeScheduler.java @@ -17,11 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Timer; import java.util.TimerTask; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Supplier; @@ -30,6 +26,7 @@ import javax.inject.Named; import javax.inject.Singleton; import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; import org.eclipse.che.api.workspace.server.hc.probe.ProbeResult.ProbeStatus; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +40,7 @@ import org.slf4j.LoggerFactory; public class ProbeScheduler { private static final Logger LOG = LoggerFactory.getLogger(ProbeScheduler.class); - private final ScheduledThreadPoolExecutor probesExecutor; + private final ScheduledExecutorService probesExecutor; /** * Use single thread for a scheduling of tasks interruption by timeout. Single thread can be used * since it is supposed that interruption is a very quick call. Separate thread is needed to @@ -55,11 +52,18 @@ public class ProbeScheduler { private final Map<String, List<ScheduledFuture>> probesFutures; @Inject - public ProbeScheduler(@Named("che.workspace.probe_pool_size") int probeSchedulerPoolSize) { + public ProbeScheduler( + @Named("che.workspace.probe_pool_size") int probeSchedulerPoolSize, + ExecutorServiceWrapper executorServiceWrapper) { probesExecutor = - new ScheduledThreadPoolExecutor( - probeSchedulerPoolSize, - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ServerProbes-%s").build()); + executorServiceWrapper.wrap( + new ScheduledThreadPoolExecutor( + probeSchedulerPoolSize, + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("ServerProbes-%s") + .build()), + ProbeScheduler.class.getName()); timeouts = new Timer("ServerProbesTimeouts", true); probesFutures = new ConcurrentHashMap<>(); } diff --git a/wsmaster/integration-tests/cascade-removal/pom.xml b/wsmaster/integration-tests/cascade-removal/pom.xml index b195d45815161f41c5de9e23308a37dae04a8b28..9af0b6c4b3301bfaf41b7677cc0cb935ba84cef7 100644 --- a/wsmaster/integration-tests/cascade-removal/pom.xml +++ b/wsmaster/integration-tests/cascade-removal/pom.xml @@ -112,6 +112,11 @@ <artifactId>che-core-commons-lang</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.eclipse.che.core</groupId> + <artifactId>che-core-commons-observability</artifactId> + <scope>test</scope> + </dependency> <dependency> <groupId>org.eclipse.che.core</groupId> <artifactId>che-core-commons-test</artifactId> diff --git a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java index 5b41d227c3a5d9edce646d660a62766fb3aaf674..2aadd7203c3f974cf7c15953a26328dd9cec7a88 100644 --- a/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java +++ b/wsmaster/integration-tests/cascade-removal/src/test/java/org/eclipse/che/core/db/jpa/CascadeRemovalTest.java @@ -97,6 +97,8 @@ import org.eclipse.che.api.workspace.server.model.impl.devfile.ProjectImpl; import org.eclipse.che.api.workspace.server.model.impl.devfile.SourceImpl; import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure; import org.eclipse.che.api.workspace.server.spi.WorkspaceDao; +import org.eclipse.che.commons.observability.ExecutorServiceWrapper; +import org.eclipse.che.commons.observability.NoopExecutorServiceWrapper; import org.eclipse.che.commons.test.db.H2DBTestServer; import org.eclipse.che.commons.test.db.PersistTestModuleBuilder; import org.eclipse.che.core.db.DBInitializer; @@ -241,6 +243,7 @@ public class CascadeRemovalTest { install(new WorkspaceJpaModule()); install(new WorkspaceActivityModule()); install(new JpaKubernetesRuntimeCacheModule()); + bind(ExecutorServiceWrapper.class).to(NoopExecutorServiceWrapper.class); bind(WorkspaceManager.class); RuntimeInfrastructure infra = mock(RuntimeInfrastructure.class); @@ -264,7 +267,9 @@ public class CascadeRemovalTest { bind(WorkspaceRuntimes.class).toInstance(wR); bind(AccountManager.class); bind(WorkspaceSharedPool.class) - .toInstance(new WorkspaceSharedPool("cached", null, null, null)); + .toInstance( + new WorkspaceSharedPool( + "cached", null, null, new NoopExecutorServiceWrapper())); MapBinder.newMapBinder(binder(), String.class, ComponentIntegrityValidator.class) .addBinding("kubernetes")