From 288209ae7186a3783245eb9450f3bdfa2a5066f4 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Mon, 28 Sep 2026 15:25:32 -0400 Subject: [PATCH 1/4] NIFI-16365 Add automatic Processor scheduling Co-authored-by: Cursor --- .../org/apache/nifi/util/NiFiProperties.java | 17 + .../main/asciidoc/administration-guide.adoc | 1 + .../nifi/jms/processors/ConsumeJMS.java | 3 + .../nifi/kafka/processors/ConsumeKafka.java | 3 + .../nifi/processors/script/ExecuteScript.java | 3 + .../script/ScriptedRecordProcessor.java | 3 + .../nifi/processors/standard/ExtractGrok.java | 3 + .../nifi/processors/standard/ExtractText.java | 3 + .../nifi/controller/queue/FlowFileQueue.java | 29 + .../queue/QueueSchedulingListener.java | 26 + .../queue/QueueSchedulingRegistration.java | 30 + .../apache/nifi/web/api/dto/ProcessorDTO.java | 13 + .../AutoSchedulingDiagnosticsDTO.java | 143 ++++ .../NodeAutoSchedulingDiagnosticsDTO.java | 64 ++ .../diagnostics/ProcessorDiagnosticsDTO.java | 20 + .../ProcessorDiagnosticsEntityMerger.java | 17 + .../ProcessorDiagnosticsEntityMergerTest.java | 92 ++ .../nifi/controller/ProcessorDetails.java | 21 + .../controller/StandardProcessorNode.java | 33 +- .../repository/StandardProcessSession.java | 64 +- .../StandardProcessSessionFactory.java | 8 +- ...tandardVersionedComponentSynchronizer.java | 18 + .../nifi/groups/StandardProcessGroup.java | 22 +- .../processor/StandardProcessContext.java | 118 +-- .../org/apache/nifi/util/ThreadUtils.java | 4 + .../controller/StandardProcessorNodeTest.java | 53 ++ .../StandardProcessSessionTest.java | 38 +- ...ardVersionedComponentSynchronizerTest.java | 36 +- .../TestVersionedComponentFlowMapper.java | 19 + .../nifi/controller/ProcessScheduler.java | 9 + .../apache/nifi/controller/ProcessorNode.java | 2 + .../scheduling/CommittedSchedulingWork.java | 32 + .../scheduling/SchedulingAgent.java | 10 + .../scheduling/SchedulingSettings.java | 36 + .../scheduling/SessionSchedulingObserver.java | 46 + .../nifi/controller/FlowController.java | 52 +- .../nifi/controller/StandardFlowSnippet.java | 17 +- .../controller/StandardReloadComponent.java | 4 +- .../StandardComponentContextProvider.java | 6 +- .../queue/AbstractFlowFileQueue.java | 21 + .../queue/StandardFlowFileQueue.java | 102 ++- .../queue/SwappablePriorityQueue.java | 6 + .../SocketLoadBalancedFlowFileQueue.java | 126 ++- .../partition/LocalQueuePartition.java | 3 + .../SwappablePriorityQueueLocalPartition.java | 6 + .../reporting/StandardReportingTaskNode.java | 9 + .../WriteAheadFlowFileRepository.java | 39 +- .../scheduling/DynamicSemaphore.java | 4 + .../scheduling/StandardProcessScheduler.java | 34 +- .../TimerDrivenSchedulingAgent.java | 20 +- .../VirtualThreadSchedulingAgent.java | 785 +++++++++++++++++- .../auto/AutoSchedulingDiagnostics.java | 193 +++++ .../auto/AutoSchedulingResetReason.java | 22 + .../auto/ConcurrencyEvaluationState.java | 22 + .../auto/ConcurrencyUpdateReason.java | 38 + .../auto/ConcurrencyUpdateStatus.java | 27 + .../auto/ProcessorSchedulingDecision.java | 31 + .../ProcessorSchedulingDecisionReason.java | 39 + .../auto/ProcessorSchedulingMeasurements.java | 221 +++++ .../auto/ProcessorSchedulingSnapshot.java | 219 +++++ .../StandardAutoSchedulingController.java | 479 +++++++++++ .../auto/SystemSchedulingMetrics.java | 168 ++++ .../auto/SystemSchedulingSnapshot.java | 145 ++++ .../controller/tasks/ConnectableTask.java | 109 ++- .../controller/tasks/ExpireFlowFiles.java | 3 +- .../controller/tasks/InvocationObserver.java | 58 ++ .../controller/tasks/InvocationOutcome.java | 29 + .../controller/tasks/InvocationResult.java | 35 +- .../nifi/controller/FlowControllerTest.java | 40 + .../controller/StandardFlowSnippetTest.java | 87 ++ .../controller/TestStandardFlowFileQueue.java | 58 ++ .../TestSocketLoadBalancedFlowFileQueue.java | 32 +- .../BatchingSessionFactoryTest.java | 3 +- .../repository/StandardProcessSessionIT.java | 19 +- .../VirtualThreadSchedulingAgentTest.java | 324 +++++++- .../ProcessorSchedulingMeasurementsTest.java | 102 +++ .../StandardAutoSchedulingControllerTest.java | 428 ++++++++++ .../auto/SystemSchedulingMetricsTest.java | 123 +++ .../controller/tasks/TestConnectableTask.java | 72 +- .../nifi-framework/nifi-resources/pom.xml | 1 + .../src/main/resources/conf/nifi.properties | 2 + .../nifi/web/StandardNiFiServiceFacade.java | 1 + .../apache/nifi/web/api/dto/DtoFactory.java | 28 + .../web/dao/impl/StandardProcessorDAO.java | 23 +- .../dao/impl/StandardReportingTaskDAO.java | 9 +- .../nifi/audit/TestProcessorAuditor.java | 6 +- .../web/StandardNiFiServiceFacadeTest.java | 11 + .../dao/impl/StandardProcessorDAOTest.java | 19 + .../edit-processor.component.html | 82 +- .../edit-processor.component.spec.ts | 66 ++ .../edit-processor.component.ts | 69 +- .../engine/StatelessFlowManager.java | 4 +- .../StatelessProcessContextFactory.java | 2 +- .../engine/StatelessReloadComponent.java | 8 +- .../stateless/flow/StandardStatelessFlow.java | 3 +- .../session/StatelessProcessSession.java | 3 +- .../system/scheduling/AutoSchedulingIT.java | 116 +++ .../AutoSchedulingPlatformFallbackIT.java | 27 + 98 files changed, 5573 insertions(+), 306 deletions(-) create mode 100644 nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/QueueSchedulingListener.java create mode 100644 nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/QueueSchedulingRegistration.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/AutoSchedulingDiagnosticsDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/NodeAutoSchedulingDiagnosticsDTO.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/ProcessorDiagnosticsEntityMergerTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/CommittedSchedulingWork.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SchedulingSettings.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SessionSchedulingObserver.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingDiagnostics.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingResetReason.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyEvaluationState.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyUpdateReason.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyUpdateStatus.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingDecision.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingDecisionReason.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurements.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingSnapshot.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingController.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingMetrics.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingSnapshot.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationObserver.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationOutcome.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/FlowControllerTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowSnippetTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurementsTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingControllerTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingMetricsTest.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingIT.java create mode 100644 nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingPlatformFallbackIT.java diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java index 12a3f01ed691..b442942e8abb 100644 --- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java @@ -85,6 +85,7 @@ public class NiFiProperties extends ApplicationProperties { public static final String BORED_YIELD_DURATION = "nifi.bored.yield.duration"; public static final String SCHEDULING_STRATEGY = "nifi.scheduling.strategy"; public static final String PROCESSOR_SCHEDULING_TIMEOUT = "nifi.processor.scheduling.timeout"; + public static final String PROCESSOR_AUTO_MAX_CONCURRENT_TASKS = "nifi.processor.auto.max.concurrent.tasks"; public static final String BACKPRESSURE_COUNT = "nifi.queue.backpressure.count"; public static final String BACKPRESSURE_SIZE = "nifi.queue.backpressure.size"; public static final String UPLOAD_WORKING_DIRECTORY = "nifi.upload.working.directory"; @@ -1477,6 +1478,22 @@ public String getSchedulingStrategy() { return getProperty(SCHEDULING_STRATEGY, DEFAULT_SCHEDULING_STRATEGY); } + public int getProcessorAutoMaxConcurrentTasks() { + final String configuredMaximum = getProperty(PROCESSOR_AUTO_MAX_CONCURRENT_TASKS, "12"); + final int maximum; + try { + maximum = Integer.parseInt(configuredMaximum.trim()); + } catch (final NumberFormatException e) { + throw new IllegalArgumentException(PROCESSOR_AUTO_MAX_CONCURRENT_TASKS + " must be a positive integer: " + configuredMaximum, e); + } + + if (maximum < 1) { + throw new IllegalArgumentException(PROCESSOR_AUTO_MAX_CONCURRENT_TASKS + " must be a positive integer: " + configuredMaximum); + } + + return (int) Math.min(maximum, 4L * Runtime.getRuntime().availableProcessors()); + } + public File getStateManagementConfigFile() { return new File(getProperty(STATE_MANAGEMENT_CONFIG_FILE, DEFAULT_STATE_MANAGEMENT_CONFIG_FILE)); } diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index 6f0401ed98bf..d457ebc13a02 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -2953,6 +2953,7 @@ This cleanup mechanism takes into account only automatically created archived _f |`nifi.administrative.yield.duration`|If a component allows an unexpected exception to escape, it is considered a bug. As a result, the framework will pause (or administratively yield) the component for this amount of time. This is done so that the component does not use up massive amounts of system resources, since it is known to have problems in the existing state. The default value is `30 secs`. |`nifi.bored.yield.duration`|When a component has no work to do (i.e., is "bored"), this is the amount of time it will wait before checking to see if it has new data to work on. This way, it does not use up CPU resources by checking for new work too often. When setting this property, be aware that it could add extra latency for components that do not constantly have work to do, as once they go into this "bored" state, they will wait this amount of time before checking for more work. The default value is `10 ms`. |`nifi.scheduling.strategy`|Selects the scheduling engine for Timer-Driven and Cron-Driven components. `AUTO` (the default) uses virtual threads on Java 25 or newer and standard scheduling on older Java versions. `VIRTUAL` always uses virtual threads. On Java 21, blocking inside synchronized component code can also block the underlying platform thread and reduce throughput. `STANDARD` uses a fixed platform thread pool sized according to the Maximum Timer Driven Thread Count configured in Controller Settings. +|`nifi.processor.auto.max.concurrent.tasks`|Sets the maximum number of concurrent tasks that a Processor using automatic scheduling is allowed to run. The default is `12` and the value must be a positive integer. NiFi also caps this value at four times the number of CPU cores available to the JVM. Automatic scheduling may use fewer concurrent tasks based on observed work and the instance's overall scheduling budget. |`nifi.queue.backpressure.count`|When drawing a new connection between two components, this is the default value for that connection's back pressure object threshold. The default is `10000` and the value must be an integer. |`nifi.queue.backpressure.size`|When drawing a new connection between two components, this is the default value for that connection's back pressure data size threshold. The default is `1 GB` and the value must be a data size including the unit of measure. |`nifi.authorizer.configuration.file`*|This is the location of the file that specifies how authorizers are defined. The default value is `./conf/authorizers.xml`. diff --git a/nifi-extension-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/ConsumeJMS.java b/nifi-extension-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/ConsumeJMS.java index 2b11e209089f..b6d55fe8bf69 100644 --- a/nifi-extension-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/ConsumeJMS.java +++ b/nifi-extension-bundles/nifi-jms-bundle/nifi-jms-processors/src/main/java/org/apache/nifi/jms/processors/ConsumeJMS.java @@ -17,6 +17,7 @@ package org.apache.nifi.jms.processors; import jakarta.jms.Session; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; @@ -68,6 +69,8 @@ * properties that came with message which are added to a {@link FlowFile} as * attributes. */ +// Durable, non-shared subscribers require exactly one configured concurrent task. +@AllowsAutoScheduling(false) @Tags({ "jms", "get", "message", "receive", "consume" }) @InputRequirement(Requirement.INPUT_FORBIDDEN) @CapabilityDescription("Consumes JMS Message of type BytesMessage, TextMessage, ObjectMessage, MapMessage or StreamMessage transforming its content to " diff --git a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java index f000ea8b8bec..088ec53ca289 100644 --- a/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java +++ b/nifi-extension-bundles/nifi-kafka-bundle/nifi-kafka-processors/src/main/java/org/apache/nifi/kafka/processors/ConsumeKafka.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.kafka.processors; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.WritesAttribute; import org.apache.nifi.annotation.behavior.WritesAttributes; @@ -131,6 +132,8 @@ }) @InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN) @SeeAlso({PublishKafka.class}) +// Changing the number of concurrent tasks changes the number of Kafka consumers. +@AllowsAutoScheduling(false) public class ConsumeKafka extends AbstractProcessor implements VerifiableProcessor, BacklogReportingProcessor { static final AllowableValue TOPIC_NAME = new AllowableValue("names", "names", "Topic is a full topic name or comma separated list of names"); diff --git a/nifi-extension-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java b/nifi-extension-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java index 0a9c8da19df7..ed0755874676 100644 --- a/nifi-extension-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java +++ b/nifi-extension-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java @@ -17,6 +17,7 @@ package org.apache.nifi.processors.script; import org.apache.commons.io.IOUtils; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; @@ -61,6 +62,8 @@ import javax.script.ScriptEngine; import javax.script.SimpleBindings; +// Script engines are created eagerly for every ProcessContext concurrency slot. +@AllowsAutoScheduling(false) @Tags({"script", "execute", "groovy", "clojure"}) @CapabilityDescription("Experimental - Executes a script given the flow file and a process session. The script is responsible for " + "handling the incoming flow file (transfer to SUCCESS or remove, e.g.) as well as any FlowFiles created by " diff --git a/nifi-extension-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedRecordProcessor.java b/nifi-extension-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedRecordProcessor.java index 3e864b1d48b4..3ed0705d5ba5 100644 --- a/nifi-extension-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedRecordProcessor.java +++ b/nifi-extension-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedRecordProcessor.java @@ -17,6 +17,7 @@ package org.apache.nifi.processors.script; import org.apache.commons.io.IOUtils; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; @@ -49,6 +50,8 @@ import javax.script.ScriptException; import javax.script.SimpleBindings; +// Script engines are created eagerly for every ProcessContext concurrency slot. +@AllowsAutoScheduling(false) abstract class ScriptedRecordProcessor extends AbstractProcessor implements Searchable { protected static final Set SCRIPT_OPTIONS = ScriptingComponentUtils.getAvailableEngines(); diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractGrok.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractGrok.java index 1d5d9029d08b..5e41cf7df2d1 100644 --- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractGrok.java +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractGrok.java @@ -22,6 +22,7 @@ import io.krakens.grok.api.GrokCompiler; import io.krakens.grok.api.Match; import io.krakens.grok.api.exception.GrokException; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.SideEffectFree; import org.apache.nifi.annotation.behavior.SupportsBatching; @@ -66,6 +67,8 @@ @SupportsBatching @SideEffectFree @InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +// A maximum-sized content buffer is allocated eagerly for every ProcessContext concurrency slot. +@AllowsAutoScheduling(false) @Tags({"grok", "log", "text", "parse", "delimit", "extract"}) @CapabilityDescription("Evaluates one or more Grok Expressions against the content of a FlowFile, " + "adding the results as attributes or replacing the content of the FlowFile with a JSON " + diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractText.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractText.java index e02a90e3f211..2c15749b6ab1 100644 --- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractText.java +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ExtractText.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.processors.standard; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; @@ -60,6 +61,8 @@ @SideEffectFree @SupportsBatching @InputRequirement(Requirement.INPUT_REQUIRED) +// A maximum-sized content buffer is allocated eagerly for every ProcessContext concurrency slot. +@AllowsAutoScheduling(false) @Tags({"evaluate", "extract", "Text", "Regular Expression", "regex"}) @CapabilityDescription( "Evaluates one or more Regular Expressions against the content of a FlowFile. " diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java index d6cd4811539b..4c2e58849c5e 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/FlowFileQueue.java @@ -25,6 +25,7 @@ import org.apache.nifi.processor.FlowFileFilter; import java.io.IOException; +import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Set; @@ -33,6 +34,25 @@ public interface FlowFileQueue { + /** + * Registers a listener for queue changes that can affect component readiness. + * + * @param listener listener to register + * @return registration used to remove the listener + */ + default QueueSchedulingRegistration addSchedulingListener(final QueueSchedulingListener listener) { + return QueueSchedulingRegistration.NO_OP; + } + + /** + * Returns the next time at which the head FlowFile can become available without another queue mutation. + * + * @return the next availability time, or {@link Instant#EPOCH} when no deadline is known + */ + default Instant getNextFlowFileAvailabilityTime() { + return Instant.EPOCH; + } + /** * @return the unique identifier for this FlowFileQueue */ @@ -97,6 +117,15 @@ public interface FlowFileQueue { QueueSize size(); + /** + * Returns work held in the local partition and available to this node. + * + * @return local queue size + */ + default QueueSize getLocalQueueSize() { + return getQueueDiagnostics().getLocalQueuePartitionDiagnostics().getActiveQueueSize(); + } + /** * Returns an atomic, point-in-time view of this queue's total {@link QueueSize} and the * FlowFiles currently held in this node's active in-memory queue. Implementations freeze every diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/QueueSchedulingListener.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/QueueSchedulingListener.java new file mode 100644 index 000000000000..3e999bdeb413 --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/QueueSchedulingListener.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.queue; + +@FunctionalInterface +public interface QueueSchedulingListener { + + /** + * Reports a queue change that can affect component readiness. + */ + void onQueueStateChanged(); +} diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/QueueSchedulingRegistration.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/QueueSchedulingRegistration.java new file mode 100644 index 000000000000..65dc7bf2c4e5 --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/QueueSchedulingRegistration.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.queue; + +@FunctionalInterface +public interface QueueSchedulingRegistration extends AutoCloseable { + + QueueSchedulingRegistration NO_OP = () -> { + }; + + /** + * Removes the registered queue scheduling listener. Repeated calls have no effect. + */ + @Override + void close(); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessorDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessorDTO.java index 475a7818720b..aab711e75e2c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessorDTO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessorDTO.java @@ -41,6 +41,7 @@ public class ProcessorDTO extends ComponentDTO { private String description; private Boolean supportsParallelProcessing; private Boolean supportsBatching; + private Boolean supportsAutoScheduling; private Boolean supportsSensitiveDynamicProperties; private Boolean supportsBacklogReporting; private Boolean persistsState; @@ -272,6 +273,18 @@ public void setSupportsBatching(Boolean supportsBatching) { this.supportsBatching = supportsBatching; } + /** + * @return whether this processor supports automatic scheduling + */ + @Schema(description = "Whether the processor supports automatic scheduling.") + public Boolean getSupportsAutoScheduling() { + return supportsAutoScheduling; + } + + public void setSupportsAutoScheduling(final Boolean supportsAutoScheduling) { + this.supportsAutoScheduling = supportsAutoScheduling; + } + /** * Gets the available relationships that this processor currently supports. * diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/AutoSchedulingDiagnosticsDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/AutoSchedulingDiagnosticsDTO.java new file mode 100644 index 000000000000..4835287c12d7 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/AutoSchedulingDiagnosticsDTO.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto.diagnostics; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "autoSchedulingDiagnostics") +public class AutoSchedulingDiagnosticsDTO { + private String executionMode; + private Integer maxConcurrentTasks; + private Integer currentConcurrentTasks; + private Integer activeProcessorInvocations; + private Long currentRunDurationMillis; + private String concurrencyEvaluationState; + private String concurrencyUpdateReason; + private Double flowFilesPerSecond; + private Long measurementWindowMillis; + private String lastConcurrencyUpdateReason; + private Boolean collectingMeasurements; + private Boolean flowFileMeasurementsAvailable; + private String concurrencyIncreaseExplanation; + + @Schema(description = "Automatic scheduling execution mode") + public String getExecutionMode() { + return executionMode; + } + + public void setExecutionMode(final String executionMode) { + this.executionMode = executionMode; + } + + public Integer getMaxConcurrentTasks() { + return maxConcurrentTasks; + } + + public void setMaxConcurrentTasks(final Integer maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + } + + public Integer getCurrentConcurrentTasks() { + return currentConcurrentTasks; + } + + public void setCurrentConcurrentTasks(final Integer currentConcurrentTasks) { + this.currentConcurrentTasks = currentConcurrentTasks; + } + + public Integer getActiveProcessorInvocations() { + return activeProcessorInvocations; + } + + public void setActiveProcessorInvocations(final Integer activeProcessorInvocations) { + this.activeProcessorInvocations = activeProcessorInvocations; + } + + public Long getCurrentRunDurationMillis() { + return currentRunDurationMillis; + } + + public void setCurrentRunDurationMillis(final Long currentRunDurationMillis) { + this.currentRunDurationMillis = currentRunDurationMillis; + } + + public String getConcurrencyEvaluationState() { + return concurrencyEvaluationState; + } + + public void setConcurrencyEvaluationState(final String concurrencyEvaluationState) { + this.concurrencyEvaluationState = concurrencyEvaluationState; + } + + public String getConcurrencyUpdateReason() { + return concurrencyUpdateReason; + } + + public void setConcurrencyUpdateReason(final String concurrencyUpdateReason) { + this.concurrencyUpdateReason = concurrencyUpdateReason; + } + + public Double getFlowFilesPerSecond() { + return flowFilesPerSecond; + } + + public void setFlowFilesPerSecond(final Double flowFilesPerSecond) { + this.flowFilesPerSecond = flowFilesPerSecond; + } + + public Long getMeasurementWindowMillis() { + return measurementWindowMillis; + } + + public void setMeasurementWindowMillis(final Long measurementWindowMillis) { + this.measurementWindowMillis = measurementWindowMillis; + } + + public String getLastConcurrencyUpdateReason() { + return lastConcurrencyUpdateReason; + } + + public void setLastConcurrencyUpdateReason(final String lastConcurrencyUpdateReason) { + this.lastConcurrencyUpdateReason = lastConcurrencyUpdateReason; + } + + public Boolean getCollectingMeasurements() { + return collectingMeasurements; + } + + public void setCollectingMeasurements(final Boolean collectingMeasurements) { + this.collectingMeasurements = collectingMeasurements; + } + + public Boolean getFlowFileMeasurementsAvailable() { + return flowFileMeasurementsAvailable; + } + + public void setFlowFileMeasurementsAvailable(final Boolean flowFileMeasurementsAvailable) { + this.flowFileMeasurementsAvailable = flowFileMeasurementsAvailable; + } + + @Schema(description = "Human-readable explanation of why automatic scheduling is not using more concurrent tasks") + public String getConcurrencyIncreaseExplanation() { + return concurrencyIncreaseExplanation; + } + + public void setConcurrencyIncreaseExplanation(final String concurrencyIncreaseExplanation) { + this.concurrencyIncreaseExplanation = concurrencyIncreaseExplanation; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/NodeAutoSchedulingDiagnosticsDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/NodeAutoSchedulingDiagnosticsDTO.java new file mode 100644 index 000000000000..2c55d95fd5b5 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/NodeAutoSchedulingDiagnosticsDTO.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto.diagnostics; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +@XmlType(name = "nodeAutoSchedulingDiagnostics") +public class NodeAutoSchedulingDiagnosticsDTO { + private String nodeId; + private String address; + private Integer apiPort; + private AutoSchedulingDiagnosticsDTO snapshot; + + @Schema(description = "Node identifier") + public String getNodeId() { + return nodeId; + } + + public void setNodeId(final String nodeId) { + this.nodeId = nodeId; + } + + @Schema(description = "Node API address") + public String getAddress() { + return address; + } + + public void setAddress(final String address) { + this.address = address; + } + + @Schema(description = "Node API port") + public Integer getApiPort() { + return apiPort; + } + + public void setApiPort(final Integer apiPort) { + this.apiPort = apiPort; + } + + @Schema(description = "Node-specific automatic scheduling details") + public AutoSchedulingDiagnosticsDTO getSnapshot() { + return snapshot; + } + + public void setSnapshot(final AutoSchedulingDiagnosticsDTO snapshot) { + this.snapshot = snapshot; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/ProcessorDiagnosticsDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/ProcessorDiagnosticsDTO.java index 89d8e4be2e20..1ea685fad633 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/ProcessorDiagnosticsDTO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/ProcessorDiagnosticsDTO.java @@ -35,6 +35,8 @@ public class ProcessorDiagnosticsDTO { private JVMDiagnosticsDTO jvmDiagnostics; private List threadDumps; private ClassLoaderDiagnosticsDTO classLoaderDiagnostics; + private AutoSchedulingDiagnosticsDTO autoSchedulingDiagnostics; + private List nodeAutoSchedulingDiagnostics; @Schema(description = "Information about the Processor for which the Diagnostic Report is generated") public ProcessorDTO getProcessor() { @@ -107,4 +109,22 @@ public ClassLoaderDiagnosticsDTO getClassLoaderDiagnostics() { public void setClassLoaderDiagnostics(ClassLoaderDiagnosticsDTO classLoaderDiagnostics) { this.classLoaderDiagnostics = classLoaderDiagnostics; } + + @Schema(description = "Automatic scheduling details for this node") + public AutoSchedulingDiagnosticsDTO getAutoSchedulingDiagnostics() { + return autoSchedulingDiagnostics; + } + + public void setAutoSchedulingDiagnostics(final AutoSchedulingDiagnosticsDTO autoSchedulingDiagnostics) { + this.autoSchedulingDiagnostics = autoSchedulingDiagnostics; + } + + @Schema(description = "Node-specific automatic scheduling details for a cluster") + public List getNodeAutoSchedulingDiagnostics() { + return nodeAutoSchedulingDiagnostics; + } + + public void setNodeAutoSchedulingDiagnostics(final List nodeAutoSchedulingDiagnostics) { + this.nodeAutoSchedulingDiagnostics = nodeAutoSchedulingDiagnostics; + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/ProcessorDiagnosticsEntityMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/ProcessorDiagnosticsEntityMerger.java index c27d3bd82958..98ecd91d99a3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/ProcessorDiagnosticsEntityMerger.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/ProcessorDiagnosticsEntityMerger.java @@ -18,11 +18,13 @@ package org.apache.nifi.cluster.manager; import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.diagnostics.AutoSchedulingDiagnosticsDTO; import org.apache.nifi.web.api.dto.diagnostics.ConnectionDiagnosticsDTO; import org.apache.nifi.web.api.dto.diagnostics.ConnectionDiagnosticsSnapshotDTO; import org.apache.nifi.web.api.dto.diagnostics.ControllerServiceDiagnosticsDTO; import org.apache.nifi.web.api.dto.diagnostics.JVMDiagnosticsSnapshotDTO; import org.apache.nifi.web.api.dto.diagnostics.LocalQueuePartitionDTO; +import org.apache.nifi.web.api.dto.diagnostics.NodeAutoSchedulingDiagnosticsDTO; import org.apache.nifi.web.api.dto.diagnostics.NodeJVMDiagnosticsSnapshotDTO; import org.apache.nifi.web.api.dto.diagnostics.ProcessorDiagnosticsDTO; import org.apache.nifi.web.api.dto.diagnostics.RemoteQueuePartitionDTO; @@ -50,6 +52,7 @@ public void mergeComponents(final ProcessorDiagnosticsEntity clientEntity, final final ProcessorDiagnosticsDTO clientDto = clientEntity.getComponent(); final List nodeJvmDiagnosticsSnapshots = new ArrayList<>(entityMap.size()); + final List nodeAutoSchedulingDiagnostics = new ArrayList<>(entityMap.size()); // Merge connection diagnostics mergeConnectionDiagnostics(clientEntity, entityMap, entity -> entity.getComponent().getIncomingConnections()); @@ -73,8 +76,22 @@ public void mergeComponents(final ProcessorDiagnosticsEntity clientEntity, final nodeJvmDiagnosticsSnapshot.setNodeId(nodeId.getId()); nodeJvmDiagnosticsSnapshot.setSnapshot(diagnosticsDto.getJvmDiagnostics().getAggregateSnapshot()); nodeJvmDiagnosticsSnapshots.add(nodeJvmDiagnosticsSnapshot); + + final AutoSchedulingDiagnosticsDTO autoSchedulingDiagnostics = diagnosticsDto.getAutoSchedulingDiagnostics(); + if (autoSchedulingDiagnostics != null) { + final NodeAutoSchedulingDiagnosticsDTO nodeDiagnostics = new NodeAutoSchedulingDiagnosticsDTO(); + nodeDiagnostics.setNodeId(nodeId.getId()); + nodeDiagnostics.setAddress(nodeId.getApiAddress()); + nodeDiagnostics.setApiPort(nodeId.getApiPort()); + nodeDiagnostics.setSnapshot(autoSchedulingDiagnostics); + nodeAutoSchedulingDiagnostics.add(nodeDiagnostics); + } } clientDto.getJvmDiagnostics().setNodeSnapshots(nodeJvmDiagnosticsSnapshots); + if (!nodeAutoSchedulingDiagnostics.isEmpty()) { + clientDto.setAutoSchedulingDiagnostics(null); + clientDto.setNodeAutoSchedulingDiagnostics(nodeAutoSchedulingDiagnostics); + } // Merge JVM Diagnostics and thread dumps final JVMDiagnosticsSnapshotDTO mergedJvmDiagnosticsSnapshot = clientDto.getJvmDiagnostics().getAggregateSnapshot().clone(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/ProcessorDiagnosticsEntityMergerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/ProcessorDiagnosticsEntityMergerTest.java new file mode 100644 index 000000000000..224a62bfee67 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/ProcessorDiagnosticsEntityMergerTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.manager; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.PermissionsDTO; +import org.apache.nifi.web.api.dto.diagnostics.AutoSchedulingDiagnosticsDTO; +import org.apache.nifi.web.api.dto.diagnostics.JVMDiagnosticsDTO; +import org.apache.nifi.web.api.dto.diagnostics.JVMDiagnosticsSnapshotDTO; +import org.apache.nifi.web.api.dto.diagnostics.NodeAutoSchedulingDiagnosticsDTO; +import org.apache.nifi.web.api.dto.diagnostics.ProcessorDiagnosticsDTO; +import org.apache.nifi.web.api.dto.status.ProcessorStatusDTO; +import org.apache.nifi.web.api.entity.ProcessorDiagnosticsEntity; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +class ProcessorDiagnosticsEntityMergerTest { + + @Test + void testPreservesNodeSpecificAutomaticSchedulingDiagnostics() { + final AutoSchedulingDiagnosticsDTO firstSnapshot = new AutoSchedulingDiagnosticsDTO(); + firstSnapshot.setCurrentConcurrentTasks(2); + final AutoSchedulingDiagnosticsDTO secondSnapshot = new AutoSchedulingDiagnosticsDTO(); + secondSnapshot.setCurrentConcurrentTasks(5); + final ProcessorDiagnosticsEntity firstEntity = createEntity(firstSnapshot); + final ProcessorDiagnosticsEntity secondEntity = createEntity(secondSnapshot); + final NodeIdentifier firstNode = createNode("node-1", "first.example", 8443); + final NodeIdentifier secondNode = createNode("node-2", "second.example", 9443); + final Map entities = new LinkedHashMap<>(); + entities.put(firstNode, firstEntity); + entities.put(secondNode, secondEntity); + + new ProcessorDiagnosticsEntityMerger(1_000L).mergeComponents(firstEntity, entities); + + final ProcessorDiagnosticsDTO merged = firstEntity.getComponent(); + assertNull(merged.getAutoSchedulingDiagnostics()); + final List nodeDiagnostics = merged.getNodeAutoSchedulingDiagnostics(); + assertEquals(2, nodeDiagnostics.size()); + assertEquals("node-1", nodeDiagnostics.get(0).getNodeId()); + assertSame(firstSnapshot, nodeDiagnostics.get(0).getSnapshot()); + assertEquals("node-2", nodeDiagnostics.get(1).getNodeId()); + assertSame(secondSnapshot, nodeDiagnostics.get(1).getSnapshot()); + } + + private ProcessorDiagnosticsEntity createEntity(final AutoSchedulingDiagnosticsDTO autoSchedulingDiagnostics) { + final ProcessorDiagnosticsDTO diagnostics = new ProcessorDiagnosticsDTO(); + diagnostics.setAutoSchedulingDiagnostics(autoSchedulingDiagnostics); + diagnostics.setIncomingConnections(Set.of()); + diagnostics.setOutgoingConnections(Set.of()); + diagnostics.setReferencedControllerServices(Set.of()); + diagnostics.setThreadDumps(new ArrayList<>()); + diagnostics.setProcessorStatus(new ProcessorStatusDTO()); + final JVMDiagnosticsDTO jvmDiagnostics = new JVMDiagnosticsDTO(); + jvmDiagnostics.setAggregateSnapshot(new JVMDiagnosticsSnapshotDTO()); + diagnostics.setJvmDiagnostics(jvmDiagnostics); + + final PermissionsDTO permissions = new PermissionsDTO(); + permissions.setCanRead(true); + final ProcessorDiagnosticsEntity entity = new ProcessorDiagnosticsEntity(); + entity.setPermissions(permissions); + entity.setComponent(diagnostics); + return entity; + } + + private NodeIdentifier createNode(final String identifier, final String address, final int apiPort) { + return new NodeIdentifier(identifier, address, 10_000, address, 10_001, address, apiPort, + address, 10_002, 10_003, true, Set.of()); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/ProcessorDetails.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/ProcessorDetails.java index edc0dbf9beac..711c9df45004 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/ProcessorDetails.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/ProcessorDetails.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.controller; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.annotation.behavior.InputRequirement; import org.apache.nifi.annotation.behavior.PrimaryNodeOnly; import org.apache.nifi.annotation.behavior.SideEffectFree; @@ -38,6 +39,7 @@ public class ProcessorDetails { private final boolean triggeredSerially; private final boolean triggerWhenAnyDestinationAvailable; private final boolean batchSupported; + private final boolean autoSchedulingSupported; private final boolean executionNodeRestricted; private final InputRequirement.Requirement inputRequirement; private final TerminationAwareLogger componentLog; @@ -52,6 +54,8 @@ public ProcessorDetails(final LoggableComponent processor) { this.triggerWhenEmpty = procClass.isAnnotationPresent(TriggerWhenEmpty.class); this.sideEffectFree = procClass.isAnnotationPresent(SideEffectFree.class); this.batchSupported = procClass.isAnnotationPresent(SupportsBatching.class); + final AllowsAutoScheduling allowsAutoScheduling = procClass.getAnnotation(AllowsAutoScheduling.class); + this.autoSchedulingSupported = allowsAutoScheduling == null || allowsAutoScheduling.value(); this.triggeredSerially = procClass.isAnnotationPresent(TriggerSerially.class); this.triggerWhenAnyDestinationAvailable = procClass.isAnnotationPresent(TriggerWhenAnyDestinationAvailable.class); this.executionNodeRestricted = procClass.isAnnotationPresent(PrimaryNodeOnly.class); @@ -92,6 +96,23 @@ public boolean isBatchSupported() { return batchSupported; } + public boolean isAutoSchedulingSupported() { + return autoSchedulingSupported; + } + + public static void verifyAutoSchedulingSupported(final Object component, final String name, final String identifier) { + if (!(component instanceof final Processor processor)) { + throw new IllegalStateException("Processor " + name + " [" + identifier + + "] cannot use scheduling strategy AUTO because its automatic scheduling capability could not be resolved"); + } + + final AllowsAutoScheduling capability = processor.getClass().getAnnotation(AllowsAutoScheduling.class); + if (capability != null && !capability.value()) { + throw new IllegalStateException("Processor " + name + " [" + identifier + + "] cannot use scheduling strategy AUTO because its implementation disables automatic scheduling"); + } + } + public boolean isExecutionNodeRestricted() { return executionNodeRestricted; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java index ca158e0b10e3..54a97ec06249 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/StandardProcessorNode.java @@ -376,6 +376,11 @@ public boolean isSessionBatchingSupported() { return processorRef.get().isBatchSupported(); } + @Override + public boolean isAutoSchedulingSupported() { + return processorRef.get().isAutoSchedulingSupported(); + } + /** * @return true if the processor has the * {@link TriggerWhenAnyDestinationAvailable} annotation, false @@ -468,6 +473,14 @@ public long getSchedulingPeriod(final TimeUnit timeUnit) { @Override public synchronized void setSchedulingStrategy(final SchedulingStrategy schedulingStrategy) { this.schedulingStrategy = schedulingStrategy; + if (schedulingStrategy == SchedulingStrategy.AUTO) { + parameterReferences.forEach(parameterReference -> decrementReferenceCounts(parameterReference.getParameterName())); + parameterReferences = Collections.emptyList(); + concurrentTaskCount.set(1); + schedulingPeriod.set("0 sec"); + schedulingNanos.set(MINIMUM_SCHEDULING_NANOS); + runNanos = 0L; + } } /** @@ -489,6 +502,12 @@ public synchronized void setSchedulingPeriod(final String schedulingPeriod) { throw new IllegalStateException("Cannot modify configuration of " + this + " while the Processor is running"); } + if (schedulingStrategy == SchedulingStrategy.AUTO) { + this.schedulingPeriod.set("0 sec"); + this.schedulingNanos.set(MINIMUM_SCHEDULING_NANOS); + return; + } + //Before setting the new Configuration references, we need to remove the current ones from reference counts. parameterReferences.forEach(parameterReference -> decrementReferenceCounts(parameterReference.getParameterName())); @@ -528,7 +547,7 @@ public synchronized void setRunDuration(final long duration, final TimeUnit time + timeUnit.toSeconds(duration) + " seconds"); } - this.runNanos = timeUnit.toNanos(duration); + this.runNanos = schedulingStrategy == SchedulingStrategy.AUTO ? 0L : timeUnit.toNanos(duration); } @Override @@ -640,7 +659,9 @@ public synchronized void setMaxConcurrentTasks(final int taskCount) { throw new IllegalArgumentException("Cannot set Concurrent Tasks to " + taskCount + " for component " + this); } - if (!isTriggeredSerially()) { + if (schedulingStrategy == SchedulingStrategy.AUTO) { + concurrentTaskCount.set(1); + } else if (!isTriggeredSerially()) { concurrentTaskCount.set(taskCount); } } @@ -1135,6 +1156,12 @@ protected Collection computeValidationErrors(final ValidationC .explanation("Processors with incoming connections cannot be scheduled for Primary Node Only.") .subject("Execution Node").valid(false).build()); } + + if (getSchedulingStrategy() == SchedulingStrategy.AUTO && !isAutoSchedulingSupported()) { + results.add(new ValidationResult.Builder() + .explanation("Processor type " + getCanonicalClassName() + " does not support automatic scheduling") + .subject("Scheduling Strategy").valid(false).build()); + } } catch (final Throwable t) { LOG.error("Failed to perform validation", t); results.add(new ValidationResult.Builder().explanation("Failed to run validation due to " + t) @@ -1224,6 +1251,8 @@ public List validateConfig(final ValidationContext validationC } } break; + case AUTO: + break; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java index 1070d8976671..17fc4741a133 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java @@ -47,6 +47,8 @@ import org.apache.nifi.controller.repository.metrics.PerformanceTracker; import org.apache.nifi.controller.repository.metrics.PerformanceTrackingInputStream; import org.apache.nifi.controller.repository.metrics.ProcessSessionEventBuilder; +import org.apache.nifi.controller.scheduling.CommittedSchedulingWork; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; import org.apache.nifi.controller.state.StandardStateMap; import org.apache.nifi.controller.status.FlowFileAvailability; import org.apache.nifi.controller.status.LoadBalanceStatus; @@ -164,6 +166,7 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn private final long sessionId; private final String connectableDescription; private final PerformanceTracker performanceTracker; + private final SessionSchedulingObserver schedulingObserver; private Map countersOnCommit; private Map immediateCounters; @@ -210,10 +213,12 @@ public class StandardProcessSession implements ProcessSession, ProvenanceEventEn private final String retryAttribute; private final FlowFileLinkage flowFileLinkage = new FlowFileLinkage(); - public StandardProcessSession(final RepositoryContext context, final TaskTermination taskTermination, final PerformanceTracker performanceTracker) { + public StandardProcessSession(final RepositoryContext context, final TaskTermination taskTermination, final PerformanceTracker performanceTracker, + final SessionSchedulingObserver schedulingObserver) { this.context = context; this.taskTermination = taskTermination; this.performanceTracker = performanceTracker; + this.schedulingObserver = Objects.requireNonNull(schedulingObserver, "Scheduling observer is required"); this.provenanceReporter = context.createProvenanceReporter(this::isFlowFileKnown, this); this.sessionId = idGenerator.getAndIncrement(); @@ -754,14 +759,30 @@ protected void commit(final Checkpoint checkpoint, final boolean asynchronous) { } } + final CommittedSchedulingWork committedSchedulingWork = schedulingObserver == SessionSchedulingObserver.NO_OP ? null : getCommittedSchedulingWork(checkpoint); + // Acknowledge records in order to update counts for incoming connections' queues acknowledgeRecords(); // Reset the internal state, now that the session has been committed resetState(); + + if (committedSchedulingWork != null) { + try { + schedulingObserver.onCommit(committedSchedulingWork); + } catch (final Throwable observerFailure) { + LOG.warn("Scheduling observer failed after Process Session commit for {}", connectableDescription, observerFailure); + } + } } catch (final Exception e) { LOG.error("Failed to commit session {}. Will roll back.", this, e); + try { + schedulingObserver.onCommitFailure(); + } catch (final Throwable observerFailure) { + LOG.warn("Scheduling observer failed after Process Session commit failure for {}", connectableDescription, observerFailure); + } + try { // if we fail to commit the session, we need to roll back // the checkpoints as well because none of the checkpoints @@ -781,6 +802,26 @@ protected void commit(final Checkpoint checkpoint, final boolean asynchronous) { } } + private CommittedSchedulingWork getCommittedSchedulingWork(final Checkpoint checkpoint) { + long committedInputFlowFiles = 0L; + long producedFlowFiles = 0L; + for (final StandardRepositoryRecord record : checkpoint.records.values()) { + final FlowFileRecord current = record.getCurrent(); + final FlowFileRecord original = record.getOriginal(); + if (original != null && record.getOriginalQueue() != null && record.getDestination() != record.getOriginalQueue()) { + committedInputFlowFiles++; + } + + final boolean committedProduction = record.getOriginalQueue() == null && !record.isMarkedForAbort() + && (!record.isMarkedForDelete() || record.getTransferRelationship() != null); + if (current != null && committedProduction) { + producedFlowFiles++; + } + } + + return new CommittedSchedulingWork(committedInputFlowFiles, producedFlowFiles); + } + private void updateEventRepository(final Checkpoint checkpoint) { try { // update event repository @@ -1821,6 +1862,8 @@ private synchronized void migrate(final StandardProcessSession newOwner, Collect } provenanceReporter.migrate(newOwner.provenanceReporter, flowFileIds); + notifySchedulingObserverOfActivity(); + newOwner.notifySchedulingObserverOfActivity(); } } @@ -2004,6 +2047,15 @@ private void registerDequeuedRecord(final FlowFileRecord flowFile, final Connect set.add(flowFile); incrementConnectionOutputCounts(connection, flowFile); + notifySchedulingObserverOfActivity(); + } + + private void notifySchedulingObserverOfActivity() { + try { + schedulingObserver.onActivity(); + } catch (final Throwable observerFailure) { + LOG.warn("Scheduling observer failed while recording Process Session activity for {}", connectableDescription, observerFailure); + } } private void handleConflictingId(final FlowFileRecord flowFile, final Connection connection, final StandardRepositoryRecord conflict) { @@ -2043,6 +2095,8 @@ public void recordGauge(final String name, final double value, final Map parents) { flowFileLinkage.addLink(flowFileId, parent.getId()); } + notifySchedulingObserverOfActivity(); return fFile; } @@ -3944,6 +4003,7 @@ private FlowFile validateRecordState(final FlowFile flowFile, final boolean allo throw new FlowFileHandlingException(flowFile + " has already been marked for removal"); } + notifySchedulingObserverOfActivity(); return record.getCurrent(); } @@ -4030,6 +4090,8 @@ private void setState(final StateMap stateMap, final Scope scope) { } else { clusterState = stateMap; } + + notifySchedulingObserverOfActivity(); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSessionFactory.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSessionFactory.java index 0c5e3fd5b293..1596b5c2aa1d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSessionFactory.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/repository/StandardProcessSessionFactory.java @@ -18,6 +18,7 @@ import org.apache.nifi.controller.lifecycle.TaskTermination; import org.apache.nifi.controller.repository.metrics.PerformanceTracker; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; import org.apache.nifi.processor.ProcessSessionFactory; public class StandardProcessSessionFactory implements ProcessSessionFactory { @@ -25,15 +26,18 @@ public class StandardProcessSessionFactory implements ProcessSessionFactory { private final RepositoryContext context; private final TaskTermination taskTermination; private final PerformanceTracker performanceTracker; + private final SessionSchedulingObserver schedulingObserver; - public StandardProcessSessionFactory(final RepositoryContext context, final TaskTermination taskTermination, final PerformanceTracker performanceTracker) { + public StandardProcessSessionFactory(final RepositoryContext context, final TaskTermination taskTermination, final PerformanceTracker performanceTracker, + final SessionSchedulingObserver schedulingObserver) { this.context = context; this.taskTermination = taskTermination; this.performanceTracker = performanceTracker; + this.schedulingObserver = schedulingObserver; } @Override public StandardProcessSession createSession() { - return new StandardProcessSession(context, taskTermination, performanceTracker); + return new StandardProcessSession(context, taskTermination, performanceTracker, schedulingObserver); } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java index e7930d84c329..bc6debcac123 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java @@ -37,6 +37,7 @@ import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; +import org.apache.nifi.controller.ProcessorDetails; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.PropertyConfiguration; import org.apache.nifi.controller.ReportingTaskNode; @@ -2821,6 +2822,10 @@ private void updateLabel(final Label label, final VersionedLabel proposed) { private ProcessorNode addProcessor(final ProcessGroup destination, final VersionedProcessor proposed, final ComponentIdGenerator componentIdGenerator, final ProcessGroup topLevelGroup) throws ProcessorInstantiationException { + if (SchedulingStrategy.AUTO.name().equals(proposed.getSchedulingStrategy())) { + verifyAutoSchedulingSupported(proposed); + } + final String identifier = componentIdGenerator.generateUuid(proposed.getIdentifier(), proposed.getInstanceIdentifier(), destination.getIdentifier()); LOG.debug("Adding Processor with ID {} of type {}", identifier, proposed.getType()); @@ -2899,6 +2904,10 @@ public void synchronize(final ProcessorNode processor, final VersionedProcessor return; } + if (proposedProcessor != null && SchedulingStrategy.AUTO.name().equals(proposedProcessor.getSchedulingStrategy())) { + verifyAutoSchedulingSupported(proposedProcessor); + } + setSynchronizationOptions(synchronizationOptions); final long timeout = System.currentTimeMillis() + synchronizationOptions.getComponentStopTimeout().toMillis(); @@ -3143,6 +3152,9 @@ private void stopControllerService(final ControllerServiceNode controllerService private void updateProcessor(final ProcessorNode processor, final VersionedProcessor proposed, final ProcessGroup topLevelGroup) throws ProcessorInstantiationException { LOG.debug("Updating Processor {}", processor); + if (SchedulingStrategy.AUTO.name().equals(proposed.getSchedulingStrategy())) { + verifyAutoSchedulingSupported(proposed); + } processor.pauseValidationTrigger(); try { @@ -3202,6 +3214,12 @@ private void updateProcessor(final ProcessorNode processor, final VersionedProce } } + private void verifyAutoSchedulingSupported(final VersionedProcessor processor) { + final BundleCoordinate coordinate = toCoordinate(processor.getBundle()); + final Object temporaryComponent = context.getExtensionManager().getTempComponent(processor.getType(), coordinate); + ProcessorDetails.verifyAutoSchedulingSupported(temporaryComponent, processor.getName(), processor.getIdentifier()); + } + private String getServiceInstanceId(final String serviceVersionedComponentId, final ProcessGroup group) { for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) { final String versionedId = serviceNode.getVersionedComponentId().orElse( diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java index eb9ae6ee06fc..d3a1dc3d932c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java @@ -729,8 +729,8 @@ private StateManager getStateManager(final ProcessorNode processorNode) { private void shutdown(final ProcessGroup procGroup) { for (final ProcessorNode node : procGroup.getProcessors()) { try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, node.getProcessor().getClass(), node.getIdentifier())) { - final StandardProcessContext processContext = new StandardProcessContext(node, controllerServiceProvider, - getStateManager(node), () -> false, nodeTypeProvider); + final StandardProcessContext processContext = StandardProcessContext.createBuilder( + node, controllerServiceProvider, getStateManager(node), () -> false, nodeTypeProvider).build(); ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnShutdown.class, node.getProcessor(), processContext); } } @@ -1292,8 +1292,8 @@ public void removeProcessor(final ProcessorNode processor) { processor.pauseValidationTrigger(); try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, processor.getProcessor().getClass(), processor.getIdentifier())) { - final StandardProcessContext processContext = new StandardProcessContext(processor, controllerServiceProvider, - getStateManager(processor), () -> false, nodeTypeProvider); + final StandardProcessContext processContext = StandardProcessContext.createBuilder( + processor, controllerServiceProvider, getStateManager(processor), () -> false, nodeTypeProvider).build(); ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnRemoved.class, processor.getProcessor(), processContext); } catch (final Exception e) { throw new ComponentLifeCycleException("Failed to invoke 'OnRemoved' methods of processor with id " + processor.getIdentifier(), e); @@ -1496,6 +1496,11 @@ public void addConnection(final Connection connection) { } finally { writeLock.unlock(); } + + scheduler.notifySchedulingEvent(connection.getSource()); + if (connection.getSource() != connection.getDestination()) { + scheduler.notifySchedulingEvent(connection.getDestination()); + } } @Override @@ -1560,6 +1565,11 @@ public void removeConnection(final Connection connectionToRemove) { } finally { writeLock.unlock(); } + + scheduler.notifySchedulingEvent(connectionToRemove.getSource()); + if (connectionToRemove.getSource() != connectionToRemove.getDestination()) { + scheduler.notifySchedulingEvent(connectionToRemove.getDestination()); + } } @Override @@ -4064,8 +4074,8 @@ public void restoreFlowPreservingIdentifiers(final VersionedExternalFlow propose private ProcessContext createProcessContext(final ProcessorNode processorNode) { final Processor processor = processorNode.getProcessor(); final Class componentClass = processor == null ? null : processor.getClass(); - return new StandardProcessContext(processorNode, controllerServiceProvider, - stateManagerProvider.getStateManager(processorNode.getIdentifier(), componentClass), () -> false, nodeTypeProvider); + return StandardProcessContext.createBuilder(processorNode, controllerServiceProvider, + stateManagerProvider.getStateManager(processorNode.getIdentifier(), componentClass), () -> false, nodeTypeProvider).build(); } private ConfigurationContext createConfigurationContext(final ComponentNode component) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardProcessContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardProcessContext.java index 25a13ebb7e51..13f075fb047b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardProcessContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardProcessContext.java @@ -60,64 +60,18 @@ public class StandardProcessContext implements ProcessContext, ControllerService private final NodeTypeProvider nodeTypeProvider; private final Map properties; private final String annotationData; + private final int maxConcurrentTasks; - public StandardProcessContext( - final ProcessorNode processorNode, - final ControllerServiceProvider controllerServiceProvider, - final StateManager stateManager, - final TaskTermination taskTermination, - final NodeTypeProvider nodeTypeProvider - ) { - - this( - processorNode, - controllerServiceProvider, - stateManager, - taskTermination, - nodeTypeProvider, - processorNode.getEffectivePropertyValues(), - processorNode.getAnnotationData() - ); - } - - public StandardProcessContext( - final ProcessorNode processorNode, - final Map propertiesOverride, - final String annotationDataOverride, - final ParameterLookup parameterLookup, - final ControllerServiceProvider controllerServiceProvider, - final StateManager stateManager, - final TaskTermination taskTermination, - final NodeTypeProvider nodeTypeProvider - ) { - this( - processorNode, - controllerServiceProvider, - stateManager, - taskTermination, - nodeTypeProvider, - resolvePropertyValues(processorNode, parameterLookup, propertiesOverride), - annotationDataOverride - ); - } - - public StandardProcessContext( - final ProcessorNode processorNode, - final ControllerServiceProvider controllerServiceProvider, - final StateManager stateManager, - final TaskTermination taskTermination, - final NodeTypeProvider nodeTypeProvider, - final Map propertyValues, - final String annotationData - ) { - this.procNode = processorNode; - this.controllerServiceProvider = controllerServiceProvider; - this.stateManager = stateManager; - this.taskTermination = taskTermination; - this.nodeTypeProvider = nodeTypeProvider; - this.annotationData = annotationData; - - properties = Collections.unmodifiableMap(propertyValues); + private StandardProcessContext(final Builder builder) { + procNode = builder.processorNode; + controllerServiceProvider = builder.controllerServiceProvider; + stateManager = builder.stateManager; + taskTermination = builder.taskTermination; + nodeTypeProvider = builder.nodeTypeProvider; + annotationData = builder.annotationData; + maxConcurrentTasks = builder.maxConcurrentTasks; + + properties = Collections.unmodifiableMap(builder.propertyValues); preparedQueries = new HashMap<>(); for (final Map.Entry entry : properties.entrySet()) { @@ -134,6 +88,54 @@ public StandardProcessContext( } } + public static Builder createBuilder(final ProcessorNode processorNode, final ControllerServiceProvider controllerServiceProvider, + final StateManager stateManager, final TaskTermination taskTermination, final NodeTypeProvider nodeTypeProvider) { + return new Builder(processorNode, controllerServiceProvider, stateManager, taskTermination, nodeTypeProvider); + } + + public static final class Builder { + private final ProcessorNode processorNode; + private final ControllerServiceProvider controllerServiceProvider; + private final StateManager stateManager; + private final TaskTermination taskTermination; + private final NodeTypeProvider nodeTypeProvider; + + private Map propertyValues; + private String annotationData; + private int maxConcurrentTasks; + + private Builder(final ProcessorNode processorNode, final ControllerServiceProvider controllerServiceProvider, + final StateManager stateManager, final TaskTermination taskTermination, final NodeTypeProvider nodeTypeProvider) { + this.processorNode = processorNode; + this.controllerServiceProvider = controllerServiceProvider; + this.stateManager = stateManager; + this.taskTermination = taskTermination; + this.nodeTypeProvider = nodeTypeProvider; + propertyValues = processorNode.getEffectivePropertyValues(); + annotationData = processorNode.getAnnotationData(); + maxConcurrentTasks = processorNode.getMaxConcurrentTasks(); + } + + public Builder setPropertyOverrides(final Map propertyOverrides, final ParameterLookup parameterLookup) { + propertyValues = resolvePropertyValues(processorNode, parameterLookup, propertyOverrides); + return this; + } + + public Builder setAnnotationData(final String annotationData) { + this.annotationData = annotationData; + return this; + } + + public Builder setMaxConcurrentTasks(final int maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + return this; + } + + public StandardProcessContext build() { + return new StandardProcessContext(this); + } + } + private static Map resolvePropertyValues(final ComponentNode component, final ParameterLookup parameterLookup, final Map propertyValues) { final Map resolvedProperties = new LinkedHashMap<>(component.getEffectivePropertyValues()); final PropertyConfigurationMapper configurationMapper = new PropertyConfigurationMapper(); @@ -219,7 +221,7 @@ public ControllerService getControllerService(final String serviceIdentifier) { @Override public int getMaxConcurrentTasks() { verifyTaskActive(); - return procNode.getMaxConcurrentTasks(); + return maxConcurrentTasks; } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/ThreadUtils.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/ThreadUtils.java index 2f3a14385a65..87173e4efbb2 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/ThreadUtils.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/util/ThreadUtils.java @@ -38,6 +38,10 @@ public static String createStackTrace(final Thread thread) { } public static String createStackTrace(final ThreadInfo threadInfo, final long[] deadlockedThreadIds, final long[] monitorDeadlockThreadIds) { + if (threadInfo == null) { + return ""; + } + final StringBuilder sb = new StringBuilder(); sb.append("\"").append(threadInfo.getThreadName()).append("\" Id="); sb.append(threadInfo.getThreadId()).append(" "); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/controller/StandardProcessorNodeTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/controller/StandardProcessorNodeTest.java index eec1c95fbebf..e0b7c771da72 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/controller/StandardProcessorNodeTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/controller/StandardProcessorNodeTest.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.controller; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.validation.ValidationTrigger; import org.apache.nifi.components.validation.VerifiableComponentFactory; @@ -25,6 +26,7 @@ import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.ProcessSessionFactory; import org.apache.nifi.processor.Processor; +import org.apache.nifi.scheduling.SchedulingStrategy; import org.apache.nifi.util.NoOpProcessor; import org.junit.jupiter.api.Test; @@ -95,10 +97,61 @@ public void onTrigger(final ProcessContext context, final ProcessSession session assertFalse(virtualThread.isAlive()); } + @Test + void testAutoSchedulingCapability() { + final ProcessScheduler processScheduler = mock(ProcessScheduler.class); + + assertTrue(createProcessorNode(new NoOpProcessor(), processScheduler).isAutoSchedulingSupported()); + assertFalse(createProcessorNode(new AutoSchedulingDisabledProcessor(), processScheduler).isAutoSchedulingSupported()); + assertFalse(createProcessorNode(new InheritedAutoSchedulingDisabledProcessor(), processScheduler).isAutoSchedulingSupported()); + assertTrue(createProcessorNode(new AutoSchedulingEnabledProcessor(), processScheduler).isAutoSchedulingSupported()); + } + + @Test + void testAutoSchedulingConfigurationIsCanonical() { + final StandardProcessorNode processorNode = createProcessorNode(new NoOpProcessor(), mock(ProcessScheduler.class)); + processorNode.setMaxConcurrentTasks(4); + processorNode.setSchedulingPeriod("10 sec"); + processorNode.setRunDuration(10L, TimeUnit.MILLISECONDS); + + processorNode.setSchedulingStrategy(SchedulingStrategy.AUTO); + assertEquals(1, processorNode.getMaxConcurrentTasks()); + assertEquals("0 sec", processorNode.getSchedulingPeriod()); + assertEquals(1L, processorNode.getSchedulingPeriod(TimeUnit.NANOSECONDS)); + assertEquals(0L, processorNode.getRunDuration(TimeUnit.MILLISECONDS)); + + processorNode.setMaxConcurrentTasks(8); + processorNode.setSchedulingPeriod("20 sec"); + processorNode.setRunDuration(20L, TimeUnit.MILLISECONDS); + assertEquals(1, processorNode.getMaxConcurrentTasks()); + assertEquals("0 sec", processorNode.getSchedulingPeriod()); + assertEquals(1L, processorNode.getSchedulingPeriod(TimeUnit.NANOSECONDS)); + assertEquals(0L, processorNode.getRunDuration(TimeUnit.MILLISECONDS)); + + processorNode.setSchedulingStrategy(SchedulingStrategy.TIMER_DRIVEN); + processorNode.setMaxConcurrentTasks(8); + processorNode.setSchedulingPeriod("20 sec"); + processorNode.setRunDuration(20L, TimeUnit.MILLISECONDS); + assertEquals(8, processorNode.getMaxConcurrentTasks()); + assertEquals("20 sec", processorNode.getSchedulingPeriod()); + assertEquals(20L, processorNode.getRunDuration(TimeUnit.MILLISECONDS)); + } + private StandardProcessorNode createProcessorNode(final Processor processor, final ProcessScheduler processScheduler) { final LoggableComponent loggableProcessor = new LoggableComponent<>(processor, BundleCoordinate.UNKNOWN_COORDINATE, null); return new StandardProcessorNode(loggableProcessor, "processor", mock(ValidationContextFactory.class), processScheduler, mock(ControllerServiceProvider.class), mock(ReloadComponent.class), mock(VerifiableComponentFactory.class), mock(ExtensionManager.class), mock(ValidationTrigger.class)); } + + @AllowsAutoScheduling(false) + private static class AutoSchedulingDisabledProcessor extends NoOpProcessor { + } + + private static class InheritedAutoSchedulingDisabledProcessor extends AutoSchedulingDisabledProcessor { + } + + @AllowsAutoScheduling + private static class AutoSchedulingEnabledProcessor extends AutoSchedulingDisabledProcessor { + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/controller/repository/StandardProcessSessionTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/controller/repository/StandardProcessSessionTest.java index ad04a8ecfd64..a60a106224f8 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/controller/repository/StandardProcessSessionTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/controller/repository/StandardProcessSessionTest.java @@ -30,6 +30,8 @@ import org.apache.nifi.controller.repository.claim.ContentClaim; import org.apache.nifi.controller.repository.claim.ContentClaimWriteCache; import org.apache.nifi.controller.repository.metrics.PerformanceTracker; +import org.apache.nifi.controller.scheduling.CommittedSchedulingWork; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; import org.apache.nifi.controller.status.FlowFileAvailability; import org.apache.nifi.controller.status.LoadBalanceStatus; import org.apache.nifi.flowfile.FlowFile; @@ -154,6 +156,9 @@ class StandardProcessSessionTest { @Captor ArgumentCaptor connectionStatusEventCaptor; + @Captor + ArgumentCaptor committedSchedulingWorkCaptor; + StandardProcessSession session; @BeforeEach @@ -165,7 +170,7 @@ void setSession() { final FlowFileActivity flowFileActivity = mock(FlowFileActivity.class); when(connectable.getFlowFileActivity()).thenReturn(flowFileActivity); - session = new StandardProcessSession(repositoryContext, taskTermination, performanceTracker); + session = new StandardProcessSession(repositoryContext, taskTermination, performanceTracker, SessionSchedulingObserver.NO_OP); } @Test @@ -190,6 +195,35 @@ void testGetTransferConnectionStatusEventsDisabled() { verify(repositoryContext, never()).recordConnectionStatusEvent(connectionStatusEventCaptor.capture()); } + @Test + void testSchedulingObserverExcludesNamedSelfLoopReturnFromCommittedInputWork() { + final SessionSchedulingObserver observer = mock(SessionSchedulingObserver.class); + session = new StandardProcessSession(repositoryContext, taskTermination, performanceTracker, observer); + setRepositoryContext(); + when(repositoryContext.getContentRepository()).thenReturn(contentRepository); + when(repositoryContext.isRecordConnectionStatusEventEnabled()).thenReturn(false); + + final Connection connection = mock(Connection.class); + final FlowFileRecord flowFileRecord = mock(FlowFileRecord.class); + final FlowFileQueue flowFileQueue = mock(FlowFileQueue.class); + when(repositoryContext.getPollableConnections()).thenReturn(List.of(connection)); + when(connection.poll(anySet())).thenReturn(flowFileRecord); + when(connection.getFlowFileQueue()).thenReturn(flowFileQueue); + when(connection.getIdentifier()).thenReturn(INPUT_CONNECTION_ID); + when(flowFileRecord.getSize()).thenReturn(EXPECTED_BYTES); + final Relationship selfLoopRelationship = new Relationship.Builder().name("retry").build(); + when(repositoryContext.getConnections(selfLoopRelationship)).thenReturn(List.of(connection)); + + final FlowFile flowFile = session.get(); + session.transfer(flowFile, selfLoopRelationship); + session.commit(); + + verify(observer).onCommit(committedSchedulingWorkCaptor.capture()); + final CommittedSchedulingWork committedWork = committedSchedulingWorkCaptor.getValue(); + assertEquals(0L, committedWork.inputFlowFiles()); + assertEquals(0L, committedWork.producedFlowFiles()); + } + @Test void testGetTransferConnectionStatusEvents() { setRepositoryContext(); @@ -304,7 +338,7 @@ void testMigrateTracksConnectionStatusEventForNewOwner() { final FlowFile flowFile = session.get(); assertNotNull(flowFile); - final StandardProcessSession newOwner = new StandardProcessSession(repositoryContext, taskTermination, performanceTracker); + final StandardProcessSession newOwner = new StandardProcessSession(repositoryContext, taskTermination, performanceTracker, SessionSchedulingObserver.NO_OP); session.migrate(newOwner); when(flowFileQueue.getBackPressureDataSizeThreshold()).thenReturn(BACK_PRESSURE_DATA_SIZE_THRESHOLD); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java index 86cdbf215bd1..79652b34bb78 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java @@ -17,6 +17,7 @@ package org.apache.nifi.flow.synchronization; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; @@ -77,6 +78,7 @@ import org.apache.nifi.parameter.StandardParameterContext; import org.apache.nifi.parameter.StandardParameterContextManager; import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.Processor; import org.apache.nifi.processor.Relationship; import org.apache.nifi.registry.flow.diff.DifferenceType; import org.apache.nifi.registry.flow.diff.FlowComparatorVersionedStrategy; @@ -213,6 +215,7 @@ public class StandardVersionedComponentSynchronizerTest { private ControllerServiceNode controllerServiceNode; private BundleCoordinate bundleCoordinate; private FlowManager flowManager; + private ExtensionManager extensionManager; private final ArgumentCaptor> propertiesCaptor = ArgumentCaptor.captor(); @@ -221,7 +224,7 @@ public class StandardVersionedComponentSynchronizerTest { @BeforeEach public void setup() { - final ExtensionManager extensionManager = mock(ExtensionManager.class); + extensionManager = mock(ExtensionManager.class); flowManager = mock(FlowManager.class); controllerServiceProvider = mock(ControllerServiceProvider.class); final Function processContextFactory = proc -> mock(ProcessContext.class); @@ -588,6 +591,26 @@ public void testSynchronizeStoppedProcessor() throws FlowSynchronizationExceptio verify(componentScheduler, times(0)).startComponent(any(Connectable.class)); } + @Test + void testSynchronizeRejectsUnresolvedAutomaticProcessorCreateBeforeMutation() { + final VersionedProcessor versionedProcessor = createUnsupportedAutomaticVersionedProcessor(); + + assertThrows(IllegalStateException.class, () -> synchronizer.synchronize(null, versionedProcessor, group, synchronizationOptions)); + verify(componentScheduler, never()).pause(); + verify(flowManager, never()).createProcessor(anyString(), anyString(), any(BundleCoordinate.class), anyBoolean()); + } + + @Test + void testSynchronizeRejectsUnsupportedAutomaticProcessorUpdateBeforeMutation() { + final VersionedProcessor versionedProcessor = createUnsupportedAutomaticVersionedProcessor(); + final BundleCoordinate coordinate = new BundleCoordinate(bundle.getGroup(), bundle.getArtifact(), bundle.getVersion()); + when(extensionManager.getTempComponent(versionedProcessor.getType(), coordinate)).thenReturn(mock(UnsupportedAutomaticProcessor.class)); + + assertThrows(IllegalStateException.class, () -> synchronizer.synchronize(processorA, versionedProcessor, group, synchronizationOptions)); + verify(componentScheduler, never()).pause(); + verify(processorA, never()).setSchedulingStrategy(any(SchedulingStrategy.class)); + } + @Test public void testSynchronizationStartsProcessor() throws FlowSynchronizationException, TimeoutException, InterruptedException { final VersionedProcessor versionedProcessor = createMinimalVersionedProcessor(); @@ -2100,6 +2123,13 @@ private VersionedProcessor createMinimalVersionedProcessor() { return versionedProcessor; } + private VersionedProcessor createUnsupportedAutomaticVersionedProcessor() { + final VersionedProcessor versionedProcessor = createMinimalVersionedProcessor(); + versionedProcessor.setType(UnsupportedAutomaticProcessor.class.getName()); + versionedProcessor.setSchedulingStrategy(SchedulingStrategy.AUTO.name()); + return versionedProcessor; + } + private VersionedControllerService createMinimalVersionedControllerService() { final VersionedControllerService versionedService = new VersionedControllerService(); versionedService.setIdentifier("12345"); @@ -2191,6 +2221,10 @@ private void assertSensitivePropertyDecrypted(final ComponentNode componentNode) private record ScheduledStateUpdate(T component, org.apache.nifi.controller.ScheduledState state) { } + @AllowsAutoScheduling(false) + private abstract static class UnsupportedAutomaticProcessor implements Processor { + } + private record ControllerServiceStateUpdate(ControllerServiceNode controllerService, ControllerServiceState state) { } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/registry/flow/mapping/TestVersionedComponentFlowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/registry/flow/mapping/TestVersionedComponentFlowMapper.java index 5916ef43cd33..899f287dc261 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/registry/flow/mapping/TestVersionedComponentFlowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/registry/flow/mapping/TestVersionedComponentFlowMapper.java @@ -66,6 +66,7 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -103,6 +104,24 @@ public void testMappingProcessorWithSensitiveValuesGivesNullValue() { assertNull(versionedProperties.get("Sensitive Property B")); } + @Test + public void testMappingAutoScheduledProcessorUsesCanonicalConfiguration() { + final VersionedComponentFlowMapper mapper = new VersionedComponentFlowMapper(mock(ExtensionManager.class), FlowMappingOptions.DEFAULT_OPTIONS); + final ProcessorNode processorNode = createProcessorNode(Collections.emptyMap()); + when(processorNode.getSchedulingStrategy()).thenReturn(SchedulingStrategy.AUTO); + when(processorNode.getMaxConcurrentTasks()).thenReturn(1); + when(processorNode.getSchedulingPeriod()).thenReturn("0 sec"); + when(processorNode.getRunDuration(TimeUnit.MILLISECONDS)).thenReturn(0L); + + final VersionedProcessor versionedProcessor = mapper.mapProcessor(processorNode, mock(ControllerServiceProvider.class), + Collections.emptySet(), Collections.emptyMap()); + + assertEquals(SchedulingStrategy.AUTO.name(), versionedProcessor.getSchedulingStrategy()); + assertEquals(1, versionedProcessor.getConcurrentlySchedulableTaskCount()); + assertEquals("0 sec", versionedProcessor.getSchedulingPeriod()); + assertEquals(0L, versionedProcessor.getRunDurationMillis()); + } + @Test public void testMappingProcessorWithSensitiveValuesLeavesSensitiveParameterReference() { final ExtensionManager extensionManager = mock(ExtensionManager.class); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessScheduler.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessScheduler.java index ebef0a18c25e..6aa45c805194 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessScheduler.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessScheduler.java @@ -18,6 +18,7 @@ import org.apache.nifi.annotation.notification.PrimaryNodeState; import org.apache.nifi.components.connector.ConnectorNode; +import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.service.ControllerServiceNode; @@ -48,6 +49,14 @@ public interface ProcessScheduler { void shutdownReportingTask(ReportingTaskNode reportingTask); + /** + * Notifies the active scheduling generation that component configuration or topology changed. + * + * @param connectable changed component + */ + default void notifySchedulingEvent(final Connectable connectable) { + } + /** * Starts scheduling the given processor to run after invoking all methods * on the underlying {@link org.apache.nifi.processor.Processor FlowFileProcessor} that diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java index b72859f75cea..fab68146c49c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java @@ -80,6 +80,8 @@ public ProcessorNode(final String id, public abstract boolean isTriggeredSerially(); + public abstract boolean isAutoSchedulingSupported(); + public abstract boolean isExecutionNodeRestricted(); public abstract Requirement getInputRequirement(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/CommittedSchedulingWork.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/CommittedSchedulingWork.java new file mode 100644 index 000000000000..1c30c48c6172 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/CommittedSchedulingWork.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling; + +/** + * Work made durable by a successful Process Session commit. + * + * @param inputFlowFiles input FlowFiles consumed + * @param producedFlowFiles FlowFiles produced before connection fan-out + */ +public record CommittedSchedulingWork(long inputFlowFiles, long producedFlowFiles) { + + public CommittedSchedulingWork { + if (inputFlowFiles < 0L || producedFlowFiles < 0L) { + throw new IllegalArgumentException("Committed scheduling work values cannot be negative"); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SchedulingAgent.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SchedulingAgent.java index e2ffce1c6c8a..ee3b83330795 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SchedulingAgent.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SchedulingAgent.java @@ -25,6 +25,16 @@ public interface SchedulingAgent { + /** + * Returns the stable concurrency ceiling exposed through the ProcessContext. + * + * @param connectable component being scheduled + * @return ProcessContext concurrency ceiling + */ + default int getProcessContextConcurrencyLimit(final Connectable connectable) { + return connectable.getMaxConcurrentTasks(); + } + void schedule(Connectable connectable, LifecycleState scheduleState); void scheduleOnce(Connectable connectable, LifecycleState scheduleState, Callable> stopCallback); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SchedulingSettings.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SchedulingSettings.java new file mode 100644 index 000000000000..83a7c083c31c --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SchedulingSettings.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling; + +/** + * Effective scheduling settings selected for a scheduling generation. + * + * @param concurrentTasks selected concurrent task count + * @param runDurationNanos selected run duration in nanoseconds + */ +public record SchedulingSettings(int concurrentTasks, long runDurationNanos) { + + public SchedulingSettings { + if (concurrentTasks < 1) { + throw new IllegalArgumentException("Concurrent tasks must be at least one"); + } + + if (runDurationNanos < 0L) { + throw new IllegalArgumentException("Run duration cannot be negative"); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SessionSchedulingObserver.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SessionSchedulingObserver.java new file mode 100644 index 000000000000..efc9ee6d78cd --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/scheduling/SessionSchedulingObserver.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling; + +/** + * Receives scheduling observations from a Process Session. + */ +public interface SessionSchedulingObserver { + + SessionSchedulingObserver NO_OP = new SessionSchedulingObserver() { + @Override + public void onActivity() { + } + + @Override + public void onCommit(final CommittedSchedulingWork work) { + } + + @Override + public void onCommitFailure() { + } + }; + + /** Records Process Session activity that prevents classifying an invocation as empty. */ + void onActivity(); + + /** Records work after a complete successful commit. */ + void onCommit(CommittedSchedulingWork work); + + /** Records a failed Process Session commit. */ + void onCommitFailure(); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 69ac6f697482..408a3032e40c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -138,6 +138,7 @@ import org.apache.nifi.controller.scheduling.StandardProcessScheduler; import org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent; import org.apache.nifi.controller.scheduling.VirtualThreadSchedulingAgent; +import org.apache.nifi.controller.scheduling.auto.AutoSchedulingDiagnostics; import org.apache.nifi.controller.serialization.FlowSerializationException; import org.apache.nifi.controller.serialization.FlowSerializer; import org.apache.nifi.controller.serialization.FlowSynchronizationException; @@ -678,10 +679,12 @@ private FlowController( flowAnalyzer.initialize(controllerServiceProvider); } + final int autoMaxConcurrentTasks = nifiProperties.getProcessorAutoMaxConcurrentTasks(); if (virtualThreadSchedulingEnabled) { - this.virtualThreadSchedulingAgent = new VirtualThreadSchedulingAgent(this, repositoryContextFactory, this.nifiProperties, maxTimerDrivenThreads.get()); + this.virtualThreadSchedulingAgent = new VirtualThreadSchedulingAgent(this, repositoryContextFactory, this.nifiProperties, maxTimerDrivenThreads.get(), autoMaxConcurrentTasks); processScheduler.setSchedulingAgent(SchedulingStrategy.TIMER_DRIVEN, virtualThreadSchedulingAgent); processScheduler.setSchedulingAgent(SchedulingStrategy.CRON_DRIVEN, virtualThreadSchedulingAgent); + processScheduler.setSchedulingAgent(SchedulingStrategy.AUTO, virtualThreadSchedulingAgent); LOG.info("Component scheduling configured to use virtual threads with a maximum of {} concurrent tasks", maxTimerDrivenThreads.get()); } else { this.virtualThreadSchedulingAgent = null; @@ -690,6 +693,7 @@ private FlowController( final CronSchedulingAgent cronSchedulingAgent = new CronSchedulingAgent(this, timerDrivenEngineRef.get(), repositoryContextFactory); processScheduler.setSchedulingAgent(SchedulingStrategy.TIMER_DRIVEN, timerDrivenAgent); processScheduler.setSchedulingAgent(SchedulingStrategy.CRON_DRIVEN, cronSchedulingAgent); + processScheduler.setSchedulingAgent(SchedulingStrategy.AUTO, timerDrivenAgent); LOG.info("Component scheduling configured to use a platform thread pool of {} threads", maxTimerDrivenThreads.get()); } @@ -1442,8 +1446,8 @@ private void notifyComponentsConfigurationRestored() { final Processor processor = procNode.getProcessor(); try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, processor.getClass(), processor.getIdentifier())) { final Class componentClass = processor == null ? null : processor.getClass(); - final StandardProcessContext processContext = new StandardProcessContext(procNode, controllerServiceProvider, - getStateManagerProvider().getStateManager(processor.getIdentifier(), componentClass), () -> false, this); + final StandardProcessContext processContext = StandardProcessContext.createBuilder(procNode, controllerServiceProvider, + getStateManagerProvider().getStateManager(processor.getIdentifier(), componentClass), () -> false, this).build(); ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnConfigurationRestored.class, processor, processContext); } } @@ -2252,6 +2256,42 @@ public int getActiveTimerDrivenThreadCount() { return virtualThreadSchedulingAgent.getActiveThreadCount(); } + public AutoSchedulingDiagnostics getAutoSchedulingDiagnostics(final ProcessorNode processorNode) { + if (processorNode.getSchedulingStrategy() != SchedulingStrategy.AUTO) { + return null; + } + + if (virtualThreadSchedulingAgent != null) { + final AutoSchedulingDiagnostics diagnostics = virtualThreadSchedulingAgent.getAutoSchedulingDiagnostics(processorNode.getIdentifier()); + if (diagnostics != null) { + return diagnostics; + } + + final int maxConcurrentTasks = processorNode.isTriggeredSerially() ? 1 : nifiProperties.getProcessorAutoMaxConcurrentTasks(); + return AutoSchedulingDiagnostics.createBuilder() + .setExecutionMode("adaptive") + .setMaxConcurrentTasks(maxConcurrentTasks) + .setCurrentConcurrentTasks(1) + .setActiveProcessorInvocations(processorNode.getActiveThreadCount()) + .setConcurrencyEvaluationState("NOT_RUNNING") + .setConcurrencyUpdateReason("PROCESSOR_NOT_RUNNING") + .setConcurrencyIncreaseExplanation("Will not increase concurrent tasks because the Processor is not running.") + .build(); + } + + final long runDurationMillis = processorNode.isSessionBatchingSupported() ? 25L : 0L; + return AutoSchedulingDiagnostics.createBuilder() + .setExecutionMode("platform fallback") + .setMaxConcurrentTasks(1) + .setCurrentConcurrentTasks(1) + .setActiveProcessorInvocations(processorNode.getActiveThreadCount()) + .setCurrentRunDurationMillis(runDurationMillis) + .setConcurrencyEvaluationState("PLATFORM_FALLBACK") + .setConcurrencyUpdateReason("PLATFORM_THREAD_FALLBACK") + .setConcurrencyIncreaseExplanation("Will not increase concurrent tasks because platform-thread fallback uses one concurrent task.") + .build(); + } + public void setMaxTimerDrivenThreadCount(final int maxThreadCount) { if (maxThreadCount < 1) { throw new IllegalArgumentException("Cannot set max number of threads to less than 1"); @@ -2354,6 +2394,12 @@ private void verifyProcessorsInVersionedFlow(final VersionedProcessGroup version if (supportedTypes.containsKey(processor.getType())) { verifyBundleInVersionedFlow(processor.getBundle(), supportedTypes.get(processor.getType())); + if (SchedulingStrategy.AUTO.name().equals(processor.getSchedulingStrategy())) { + final Bundle bundle = processor.getBundle(); + final BundleCoordinate coordinate = new BundleCoordinate(bundle.getGroup(), bundle.getArtifact(), bundle.getVersion()); + final Object temporaryComponent = extensionManager.getTempComponent(processor.getType(), coordinate); + ProcessorDetails.verifyAutoSchedulingSupported(temporaryComponent, processor.getName(), processor.getIdentifier()); + } } else { throw new IllegalStateException("Invalid Processor Type: " + processor.getType()); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java index 34f6ad108927..f286c1d37edc 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java @@ -209,6 +209,7 @@ private void verifyProcessorsInSnippet(final FlowSnippetDTO templateContents, fi if (supportedTypes.containsKey(processor.getType())) { verifyBundleInSnippet(processor.getBundle(), supportedTypes.get(processor.getType())); + verifyAutoSchedulingSupported(processor); } else { throw new IllegalStateException("Invalid Processor Type: " + processor.getType()); } @@ -220,6 +221,18 @@ private void verifyProcessorsInSnippet(final FlowSnippetDTO templateContents, fi } } + private void verifyAutoSchedulingSupported(final ProcessorDTO processor) { + final ProcessorConfigDTO config = processor.getConfig(); + if (config == null || !SchedulingStrategy.AUTO.name().equals(config.getSchedulingStrategy())) { + return; + } + + final BundleDTO bundle = processor.getBundle(); + final BundleCoordinate coordinate = new BundleCoordinate(bundle.getGroup(), bundle.getArtifact(), bundle.getVersion()); + final Object temporaryComponent = extensionManager.getTempComponent(processor.getType(), coordinate); + ProcessorDetails.verifyAutoSchedulingSupported(temporaryComponent, processor.getName(), processor.getId()); + } + public void instantiate(final FlowManager flowManager, final FlowController flowController, final ProcessGroup group, final boolean topLevel) { // // Instantiate Controller Services @@ -422,8 +435,8 @@ public void instantiate(final FlowManager flowManager, final FlowController flow // Notify the processor node that the configuration (properties, e.g.) has been restored final Class componentClass = procNode.getProcessor() == null ? null : procNode.getProcessor().getClass(); - final StandardProcessContext processContext = new StandardProcessContext(procNode, flowController.getControllerServiceProvider(), - flowController.getStateManagerProvider().getStateManager(procNode.getProcessor().getIdentifier(), componentClass), () -> false, flowController); + final StandardProcessContext processContext = StandardProcessContext.createBuilder(procNode, flowController.getControllerServiceProvider(), + flowController.getStateManagerProvider().getStateManager(procNode.getProcessor().getIdentifier(), componentClass), () -> false, flowController).build(); procNode.onConfigurationRestored(processContext); } finally { procNode.resumeValidationTrigger(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardReloadComponent.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardReloadComponent.java index 4ca87b21c05d..f4ddbc433bb0 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardReloadComponent.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardReloadComponent.java @@ -78,8 +78,8 @@ public void reload(final ProcessorNode existingNode, final String newType, final final Class componentClass = existingNode.getProcessor() == null ? null : existingNode.getProcessor().getClass(); final StateManager stateManager = flowController.getStateManagerProvider().getStateManager(id, componentClass); - final StandardProcessContext processContext = new StandardProcessContext(existingNode, flowController.getControllerServiceProvider(), - stateManager, () -> false, flowController); + final StandardProcessContext processContext = StandardProcessContext.createBuilder( + existingNode, flowController.getControllerServiceProvider(), stateManager, () -> false, flowController).build(); // Cleanup the URL ClassLoader for the existing processor instance. extensionManager.closeURLClassLoader(id, existingInstanceClassLoader); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardComponentContextProvider.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardComponentContextProvider.java index 6e7973ba0200..25eb6219386b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardComponentContextProvider.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardComponentContextProvider.java @@ -54,8 +54,10 @@ public ProcessContext createProcessContext(final ProcessorNode processorNode, fi final StateManager stateManager = flowController.getStateManagerProvider().getStateManager(processorNode.getIdentifier()); final TaskTermination taskTermination = () -> false; final Map serviceReferencedProperties = resolveServiceReferences(processorNode, processorNode.getProcessGroup(), propertiesOverride); - return new StandardProcessContext(processorNode, serviceReferencedProperties, null, parameterLookup, - flowController.getControllerServiceProvider(), stateManager, taskTermination, flowController); + return StandardProcessContext.createBuilder(processorNode, flowController.getControllerServiceProvider(), stateManager, taskTermination, flowController) + .setPropertyOverrides(serviceReferencedProperties, parameterLookup) + .setAnnotationData(null) + .build(); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/AbstractFlowFileQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/AbstractFlowFileQueue.java index df7a403a8a99..a53b1adae7c0 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/AbstractFlowFileQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/AbstractFlowFileQueue.java @@ -61,6 +61,7 @@ public abstract class AbstractFlowFileQueue implements FlowFileQueue { private final ConcurrentMap listRequestMap = new ConcurrentHashMap<>(); private final ConcurrentMap dropRequestMap = new ConcurrentHashMap<>(); + private final Set schedulingListeners = ConcurrentHashMap.newKeySet(); private LoadBalanceStrategy loadBalanceStrategy = LoadBalanceStrategy.DO_NOT_LOAD_BALANCE; private String partitioningAttribute = null; @@ -88,6 +89,22 @@ protected ProcessScheduler getScheduler() { return scheduler; } + @Override + public QueueSchedulingRegistration addSchedulingListener(final QueueSchedulingListener listener) { + schedulingListeners.add(listener); + return () -> schedulingListeners.remove(listener); + } + + protected void notifySchedulingListeners() { + for (final QueueSchedulingListener listener : schedulingListeners) { + try { + listener.onQueueStateChanged(); + } catch (final Throwable listenerFailure) { + logger.warn("Queue scheduling listener failed for {}", identifier, listenerFailure); + } + } + } + @Override public String getFlowFileExpiration() { return expirationPeriod.get().getPeriod(); @@ -116,6 +133,8 @@ public void setBackPressureObjectThreshold(final long threshold) { final MaxQueueSize updatedSize = new MaxQueueSize(maxSize.getMaxSize(), maxSize.getMaxBytes(), threshold); updated = maxQueueSize.compareAndSet(maxSize, updatedSize); } + + notifySchedulingListeners(); } @Override @@ -133,6 +152,8 @@ public void setBackPressureDataSizeThreshold(final String maxDataSize) { final MaxQueueSize updatedSize = new MaxQueueSize(maxDataSize, maxBytes, maxSize.getMaxCount()); updated = maxQueueSize.compareAndSet(maxSize, updatedSize); } + + notifySchedulingListeners(); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/StandardFlowFileQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/StandardFlowFileQueue.java index cbfe04cab2dd..d8a8be4c1c1a 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/StandardFlowFileQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/StandardFlowFileQueue.java @@ -35,6 +35,7 @@ import org.apache.nifi.util.concurrency.TimedLock; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -97,6 +98,7 @@ public boolean isActivelyLoadBalancing() { @Override public void setPriorities(final List newPriorities) { queue.setPriorities(newPriorities); + notifySchedulingListeners(); } @Override @@ -117,33 +119,51 @@ public QueueDiagnostics getQueueDiagnostics() { @Override public void put(final FlowFileRecord file) { queue.put(file); + notifySchedulingListeners(); } @Override public void putAll(final Collection files) { queue.putAll(files); + if (!files.isEmpty()) { + notifySchedulingListeners(); + } } @Override public FlowFileRecord poll(final Set expiredRecords, final PollStrategy pollStrategy) { // First check if we have any records Pre-Fetched. final long expirationMillis = getFlowFileExpiration(TimeUnit.MILLISECONDS); - return queue.poll(expiredRecords, expirationMillis, pollStrategy); + final FlowFileRecord flowFile = queue.poll(expiredRecords, expirationMillis, pollStrategy); + if (flowFile != null || !expiredRecords.isEmpty()) { + notifySchedulingListeners(); + } + + return flowFile; } @Override public List poll(int maxResults, final Set expiredRecords, final PollStrategy pollStrategy) { - return queue.poll(maxResults, expiredRecords, getFlowFileExpiration(TimeUnit.MILLISECONDS), pollStrategy); + final List flowFiles = queue.poll(maxResults, expiredRecords, getFlowFileExpiration(TimeUnit.MILLISECONDS), pollStrategy); + if (!flowFiles.isEmpty() || !expiredRecords.isEmpty()) { + notifySchedulingListeners(); + } + + return flowFiles; } @Override public void acknowledge(final FlowFileRecord flowFile) { queue.acknowledge(flowFile); + notifySchedulingListeners(); } @Override public void acknowledge(final Collection flowFiles) { queue.acknowledge(flowFiles); + if (!flowFiles.isEmpty()) { + notifySchedulingListeners(); + } } @Override @@ -181,6 +201,11 @@ public FlowFileAvailability getFlowFileAvailability() { return queue.getFlowFileAvailability(); } + @Override + public Instant getNextFlowFileAvailabilityTime() { + return queue.getNextFlowFileAvailabilityTime(); + } + @Override public boolean isActiveQueueEmpty() { final FlowFileQueueSize queueSize = queue.getFlowFileQueueSize(); @@ -189,7 +214,12 @@ public boolean isActiveQueueEmpty() { @Override public List poll(final FlowFileFilter filter, final Set expiredRecords, final PollStrategy pollStrategy) { - return queue.poll(filter, expiredRecords, getFlowFileExpiration(TimeUnit.MILLISECONDS), pollStrategy); + final List flowFiles = queue.poll(filter, expiredRecords, getFlowFileExpiration(TimeUnit.MILLISECONDS), pollStrategy); + if (!flowFiles.isEmpty() || !expiredRecords.isEmpty()) { + notifySchedulingListeners(); + } + + return flowFiles; } @Override @@ -215,54 +245,62 @@ public FlowFileRecord getFlowFile(final String flowFileUuid) throws IOException @Override protected void dropFlowFiles(final DropFlowFileRequest dropRequest, final String requestor) { queue.dropFlowFiles(dropRequest, requestor); + notifySchedulingListeners(); } @Override public DropFlowFileSummary dropFlowFiles(final Predicate predicate) throws IOException { + final DropFlowFileSummary summary; lock(); try { // Perform the selective drop on the queue, which returns the dropped FlowFiles and swap location updates final SelectiveDropResult dropResult = queue.dropFlowFiles(predicate); if (dropResult.getDroppedCount() == 0) { - return new DropFlowFileSummary(0, 0L); - } + summary = new DropFlowFileSummary(0, 0L); + } else { + // Create repository records for the dropped FlowFiles + final List droppedFlowFiles = dropResult.getDroppedFlowFiles(); + final List repositoryRecords = new ArrayList<>(createDeleteRepositoryRecords(droppedFlowFiles)); + + // Create repository records for swap file changes so the FlowFile Repository can track valid swap locations + for (final Map.Entry entry : dropResult.getSwapLocationUpdates().entrySet()) { + final String oldSwapLocation = entry.getKey(); + final String newSwapLocation = entry.getValue(); + + final StandardRepositoryRecord swapRecord = new StandardRepositoryRecord(this); + if (newSwapLocation == null) { + swapRecord.setSwapLocation(oldSwapLocation, RepositoryRecordType.SWAP_FILE_DELETED); + } else { + swapRecord.setSwapFileRenamed(oldSwapLocation, newSwapLocation); + } + repositoryRecords.add(swapRecord); + } - // Create repository records for the dropped FlowFiles - final List droppedFlowFiles = dropResult.getDroppedFlowFiles(); - final List repositoryRecords = new ArrayList<>(createDeleteRepositoryRecords(droppedFlowFiles)); + // Update the FlowFile Repository + getFlowFileRepository().updateRepository(repositoryRecords); - // Create repository records for swap file changes so the FlowFile Repository can track valid swap locations - for (final Map.Entry entry : dropResult.getSwapLocationUpdates().entrySet()) { - final String oldSwapLocation = entry.getKey(); - final String newSwapLocation = entry.getValue(); + // Create and register provenance events + final List provenanceEvents = createDropProvenanceEvents(droppedFlowFiles, "Selective drop by predicate"); + getProvenanceRepository().registerEvents(provenanceEvents); - final StandardRepositoryRecord swapRecord = new StandardRepositoryRecord(this); - if (newSwapLocation == null) { - swapRecord.setSwapLocation(oldSwapLocation, RepositoryRecordType.SWAP_FILE_DELETED); - } else { - swapRecord.setSwapFileRenamed(oldSwapLocation, newSwapLocation); + // Delete old swap files that were replaced + for (final Map.Entry entry : dropResult.getSwapLocationUpdates().entrySet()) { + final String oldSwapLocation = entry.getKey(); + swapManager.deleteSwapFile(oldSwapLocation); } - repositoryRecords.add(swapRecord); - } - - // Update the FlowFile Repository - getFlowFileRepository().updateRepository(repositoryRecords); - // Create and register provenance events - final List provenanceEvents = createDropProvenanceEvents(droppedFlowFiles, "Selective drop by predicate"); - getProvenanceRepository().registerEvents(provenanceEvents); - - // Delete old swap files that were replaced - for (final Map.Entry entry : dropResult.getSwapLocationUpdates().entrySet()) { - final String oldSwapLocation = entry.getKey(); - swapManager.deleteSwapFile(oldSwapLocation); + summary = new DropFlowFileSummary(dropResult.getDroppedCount(), dropResult.getDroppedBytes()); } - - return new DropFlowFileSummary(dropResult.getDroppedCount(), dropResult.getDroppedBytes()); } finally { unlock(); } + + if (summary.getDroppedCount() > 0) { + notifySchedulingListeners(); + } + + return summary; } /** diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java index dee5c139bf56..62cea7114923 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/SwappablePriorityQueue.java @@ -38,6 +38,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -511,6 +512,11 @@ public FlowFileAvailability getFlowFileAvailability() { return FlowFileAvailability.FLOWFILE_AVAILABLE; } + public Instant getNextFlowFileAvailabilityTime() { + final long expiration = topPenaltyExpiration; + return expiration > System.currentTimeMillis() ? Instant.ofEpochMilli(expiration) : Instant.EPOCH; + } + public void acknowledge(final FlowFileRecord flowFile) { logger.trace("{} Acknowledging {}", this, flowFile); directlyIncrementUnacknowledgedQueueSize(-1, -flowFile.getSize()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/SocketLoadBalancedFlowFileQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/SocketLoadBalancedFlowFileQueue.java index 4985cd58fe9a..db796ee07237 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/SocketLoadBalancedFlowFileQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/SocketLoadBalancedFlowFileQueue.java @@ -79,6 +79,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -200,6 +201,8 @@ public synchronized void setLoadBalanceStrategy(final LoadBalanceStrategy strate return; } + notifySchedulingListeners(); + if (clusterCoordinator == null) { // Not clustered so nothing to worry about. return; @@ -256,7 +259,7 @@ public void offloadQueue() { } if (!nodesToKeep.isEmpty()) { - setNodeIdentifiers(nodesToKeep, false); + setNodeIdentifiers(nodesToKeep, false, false); } // Update our partitioner so that we don't keep any data on the local partition @@ -269,6 +272,7 @@ public void offloadQueue() { } finally { partitionWriteLock.unlock(); } + notifySchedulingListeners(); } private Map getPartitionSizes() { @@ -302,6 +306,7 @@ public void resetOffloadedQueue() { setFlowFilePartitioner(partitioner); logger.debug("Queue {} is no longer offloaded, restored load balance strategy to {} and partitioning attribute to \"{}\"", this, getLoadBalanceStrategy(), getPartitioningAttribute()); + notifySchedulingListeners(); } } @@ -427,6 +432,8 @@ public synchronized void setPriorities(final List newPriori } finally { partitionReadLock.unlock(); } + + notifySchedulingListeners(); } @Override @@ -546,6 +553,11 @@ public QueueSize size() { return totalSize.get(); } + @Override + public QueueSize getLocalQueueSize() { + return localPartition.getQueueDiagnostics().getActiveQueueSize(); + } + /** * Returns an atomic, point-in-time snapshot of this queue by freezing every partition on this node for the * duration of the call. The {@code partitionReadLock} prevents cluster-topology changes from replacing the @@ -618,6 +630,11 @@ public FlowFileAvailability getFlowFileAvailability() { return localPartition.getFlowFileAvailability(); } + @Override + public Instant getNextFlowFileAvailabilityTime() { + return localPartition.getNextFlowFileAvailabilityTime(); + } + @Override public boolean isActiveQueueEmpty() { return localPartition.isActiveQueueEmpty(); @@ -682,6 +699,9 @@ private void adjustSize(final int countToAdd, final long bytesToAdd) { @Override public void onTransfer(final Collection flowFiles) { adjustSize(-flowFiles.size(), -flowFiles.stream().mapToLong(FlowFileRecord::getSize).sum()); + if (!flowFiles.isEmpty()) { + notifySchedulingListeners(); + } } @Override @@ -691,6 +711,7 @@ public void onAbort(final Collection flowFiles) { } adjustSize(-flowFiles.size(), -flowFiles.stream().mapToLong(FlowFileRecord::getSize).sum()); + notifySchedulingListeners(); } @Override @@ -711,6 +732,10 @@ private QueuePartition getPartition(final FlowFileRecord flowFile) { } public void setNodeIdentifiers(final Set updatedNodeIdentifiers, final boolean forceUpdate) { + setNodeIdentifiers(updatedNodeIdentifiers, forceUpdate, true); + } + + private void setNodeIdentifiers(final Set updatedNodeIdentifiers, final boolean forceUpdate, final boolean notifySchedulingListenersAfterUpdate) { partitionWriteLock.lock(); try { // If nothing is changing, then just return @@ -831,6 +856,10 @@ public void setNodeIdentifiers(final Set updatedNodeIdentifiers, } finally { partitionWriteLock.unlock(); } + + if (notifySchedulingListenersAfterUpdate) { + notifySchedulingListeners(); + } } protected void rebalance(final QueuePartition partition) { @@ -842,6 +871,7 @@ protected void rebalance(final QueuePartition partition) { @Override public void put(final FlowFileRecord flowFile) { putAndGetPartition(flowFile); + notifySchedulingListeners(); } protected QueuePartition putAndGetPartition(final FlowFileRecord flowFile) { @@ -873,7 +903,7 @@ public void receiveFromPeer(final Collection flowFiles) throws I if (partitioner.isRebalanceOnClusterResize()) { logger.debug("Received the following FlowFiles from Peer: {}. Will re-partition FlowFiles to ensure proper balancing across the cluster.", flowFiles); - putAll(flowFiles); + putAll(flowFiles, false); } else { logger.debug("Received the following FlowFiles from Peer: {}. Will accept FlowFiles to the local partition", flowFiles); @@ -900,11 +930,22 @@ public void receiveFromPeer(final Collection flowFiles) throws I } finally { partitionReadLock.unlock(); } + + if (!flowFiles.isEmpty()) { + notifySchedulingListeners(); + } } @Override public void putAll(final Collection flowFiles) { + putAll(flowFiles, true); + } + + private void putAll(final Collection flowFiles, final boolean notifySchedulingListenersAfterUpdate) { putAllAndGetPartitions(flowFiles); + if (notifySchedulingListenersAfterUpdate && !flowFiles.isEmpty()) { + notifySchedulingListeners(); + } } protected Map> putAllAndGetPartitions(final Collection flowFiles) { @@ -990,6 +1031,10 @@ protected void setFlowFilePartitioner(final FlowFilePartitioner partitioner) { public FlowFileRecord poll(final Set expiredRecords, final PollStrategy pollStrategy) { final FlowFileRecord flowFile = localPartition.poll(expiredRecords, pollStrategy); onAbort(expiredRecords); + if (flowFile != null) { + notifySchedulingListeners(); + } + return flowFile; } @@ -997,6 +1042,10 @@ public FlowFileRecord poll(final Set expiredRecords, final PollS public List poll(int maxResults, Set expiredRecords, final PollStrategy pollStrategy) { final List flowFiles = localPartition.poll(maxResults, expiredRecords, pollStrategy); onAbort(expiredRecords); + if (!flowFiles.isEmpty()) { + notifySchedulingListeners(); + } + return flowFiles; } @@ -1004,6 +1053,10 @@ public List poll(int maxResults, Set expiredReco public List poll(FlowFileFilter filter, Set expiredRecords, final PollStrategy pollStrategy) { final List flowFiles = localPartition.poll(filter, expiredRecords, pollStrategy); onAbort(expiredRecords); + if (!flowFiles.isEmpty()) { + notifySchedulingListeners(); + } + return flowFiles; } @@ -1012,6 +1065,7 @@ public void acknowledge(final FlowFileRecord flowFile) { localPartition.acknowledge(flowFile); adjustSize(-1, -flowFile.getSize()); + notifySchedulingListeners(); } @Override @@ -1021,6 +1075,7 @@ public void acknowledge(final Collection flowFiles) { if (!flowFiles.isEmpty()) { final long bytes = flowFiles.stream().mapToLong(FlowFileRecord::getSize).sum(); adjustSize(-flowFiles.size(), -bytes); + notifySchedulingListeners(); } } @@ -1142,6 +1197,7 @@ protected void dropFlowFiles(final DropFlowFileRequest dropRequest, final String } finally { partitionReadLock.unlock(); } + notifySchedulingListeners(); } @Override @@ -1202,6 +1258,7 @@ public DropFlowFileSummary dropFlowFiles(final Predicate predicate) th return new DropFlowFileSummary(totalDroppedCount, totalDroppedBytes); } finally { unlock(); + notifySchedulingListeners(); } } @@ -1243,10 +1300,12 @@ public void onNodeAdded(final NodeIdentifier nodeId) { updatedNodeIds.add(nodeId); logger.debug("Node Identifier {} added to cluster. Node ID's changing from {} to {}", nodeId, nodeIdentifiers, updatedNodeIds); - setNodeIdentifiers(updatedNodeIds, false); + setNodeIdentifiers(updatedNodeIds, false, false); } finally { partitionWriteLock.unlock(); } + + notifySchedulingListeners(); } @Override @@ -1260,10 +1319,12 @@ public void onNodeRemoved(final NodeIdentifier nodeId) { } logger.debug("Node Identifier {} removed from cluster. Node ID's changing from {} to {}", nodeId, nodeIdentifiers, updatedNodeIds); - setNodeIdentifiers(updatedNodeIds, false); + setNodeIdentifiers(updatedNodeIds, false, false); } finally { partitionWriteLock.unlock(); } + + notifySchedulingListeners(); } @Override @@ -1279,7 +1340,7 @@ public void onLocalNodeIdentifierSet(final NodeIdentifier localNodeId) { updatedNodeIds.add(localNodeId); logger.debug("Local Node Identifier has now been determined to be {}. Adding to set of Node Identifiers for {}", localNodeId, SocketLoadBalancedFlowFileQueue.this); - setNodeIdentifiers(updatedNodeIds, false); + setNodeIdentifiers(updatedNodeIds, false, false); } logger.debug("Local Node Identifier set to {}; current partitions = {}", localNodeId, queuePartitions); @@ -1303,7 +1364,7 @@ public void onLocalNodeIdentifierSet(final NodeIdentifier localNodeId) { final Set updatedNodeIds = new HashSet<>(nodeIdentifiers); updatedNodeIds.add(localNodeId); - setNodeIdentifiers(updatedNodeIds, true); + setNodeIdentifiers(updatedNodeIds, true, false); return; } } @@ -1312,39 +1373,36 @@ public void onLocalNodeIdentifierSet(final NodeIdentifier localNodeId) { } finally { partitionWriteLock.unlock(); } + + notifySchedulingListeners(); } @Override public void onNodeStateChange(final NodeIdentifier nodeId, final NodeConnectionState newState) { - partitionWriteLock.lock(); - try { - if (!offloaded) { - switch (newState) { - case OFFLOADING: - onNodeRemoved(nodeId); - break; - case CONNECTED: - onNodeAdded(nodeId); - break; - } - } else { - switch (newState) { - case CONNECTED: - if (nodeId != null && nodeId.equals(clusterCoordinator.getLocalNodeIdentifier())) { - // the node with this queue was connected to the cluster, make sure the queue is not offloaded - resetOffloadedQueue(); - } - break; - case OFFLOADED: - case OFFLOADING: - case DISCONNECTED: - case DISCONNECTING: - onNodeRemoved(nodeId); - break; - } + if (!offloaded) { + switch (newState) { + case OFFLOADING: + onNodeRemoved(nodeId); + break; + case CONNECTED: + onNodeAdded(nodeId); + break; + } + } else { + switch (newState) { + case CONNECTED: + if (nodeId != null && nodeId.equals(clusterCoordinator.getLocalNodeIdentifier())) { + // the node with this queue was connected to the cluster, make sure the queue is not offloaded + resetOffloadedQueue(); + } + break; + case OFFLOADED: + case OFFLOADING: + case DISCONNECTED: + case DISCONNECTING: + onNodeRemoved(nodeId); + break; } - } finally { - partitionWriteLock.unlock(); } } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/LocalQueuePartition.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/LocalQueuePartition.java index bd7a7ecab48a..7aa407ba94ac 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/LocalQueuePartition.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/LocalQueuePartition.java @@ -26,6 +26,7 @@ import org.apache.nifi.processor.FlowFileFilter; import java.io.IOException; +import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Set; @@ -50,6 +51,8 @@ public interface LocalQueuePartition extends QueuePartition { */ boolean isUnacknowledgedFlowFile(); + Instant getNextFlowFileAvailabilityTime(); + /** * Returns a single FlowFile with the highest priority that is available in the partition, or null if no FlowFile is available * diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/SwappablePriorityQueueLocalPartition.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/SwappablePriorityQueueLocalPartition.java index e6a57bca32f1..79394f01877d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/SwappablePriorityQueueLocalPartition.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/queue/clustered/partition/SwappablePriorityQueueLocalPartition.java @@ -38,6 +38,7 @@ import org.apache.nifi.processor.FlowFileFilter; import java.io.IOException; +import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -87,6 +88,11 @@ public boolean isUnacknowledgedFlowFile() { return priorityQueue.isUnacknowledgedFlowFile(); } + @Override + public Instant getNextFlowFileAvailabilityTime() { + return priorityQueue.getNextFlowFileAvailabilityTime(); + } + @Override public Optional getNodeIdentifier() { return Optional.empty(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/reporting/StandardReportingTaskNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/reporting/StandardReportingTaskNode.java index 3655d7c6b707..36a042cb54c4 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/reporting/StandardReportingTaskNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/reporting/StandardReportingTaskNode.java @@ -35,6 +35,7 @@ import org.apache.nifi.parameter.ParameterLookup; import org.apache.nifi.reporting.ReportingContext; import org.apache.nifi.reporting.ReportingTask; +import org.apache.nifi.scheduling.SchedulingStrategy; import java.util.Collections; import java.util.List; @@ -87,6 +88,14 @@ public ReportingContext getReportingContext() { @Override protected List validateConfig(final ValidationContext validationContext) { + if (getSchedulingStrategy() == SchedulingStrategy.AUTO) { + return List.of(new ValidationResult.Builder() + .subject("Scheduling Strategy") + .explanation("Reporting Tasks do not support automatic scheduling") + .valid(false) + .build()); + } + return Collections.emptyList(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java index f18d5b89159d..fa9b5b750462 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlowFileRepository.java @@ -704,26 +704,37 @@ public void onSync(final int partitionIndex) { @Override public void onGlobalSync() { - for (final BlockingQueue claimQueue : claimsAwaitingDestruction.values()) { - final Set claimsToDestroy = new HashSet<>(); - claimQueue.drainTo(claimsToDestroy); + final long startedNanos = System.nanoTime(); - for (final ResourceClaim claim : claimsToDestroy) { - markDestructable(claim); + try { + for (final BlockingQueue claimQueue : claimsAwaitingDestruction.values()) { + final Set claimsToDestroy = new HashSet<>(); + claimQueue.drainTo(claimsToDestroy); + + for (final ResourceClaim claim : claimsToDestroy) { + markDestructable(claim); + } } - } - for (final BlockingQueue claimQueue : claimsAwaitingTruncation.values()) { - final Set claimsToTruncate = new HashSet<>(); - claimQueue.drainTo(claimsToTruncate); + for (final BlockingQueue claimQueue : claimsAwaitingTruncation.values()) { + final Set claimsToTruncate = new HashSet<>(); + claimQueue.drainTo(claimsToTruncate); - for (final ContentClaim claim : claimsToTruncate) { - if (isTruncationAllowed(claim)) { - claimManager.markTruncatable(claim); - } else { - logger.debug("Skipping markTruncatable for {} during onGlobalSync because truncation is no longer allowed", claim); + for (final ContentClaim claim : claimsToTruncate) { + if (isTruncationAllowed(claim)) { + claimManager.markTruncatable(claim); + } else { + logger.debug("Skipping markTruncatable for {} during onGlobalSync because truncation is no longer allowed", claim); + } } } + } finally { + final long elapsedNanos = System.nanoTime() - startedNanos; + + if (elapsedNanos >= TimeUnit.SECONDS.toNanos(1)) { + logger.info("Content Repository cleanup delayed FlowFile Repository synchronization by {} ms", + TimeUnit.NANOSECONDS.toMillis(elapsedNanos)); + } } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/DynamicSemaphore.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/DynamicSemaphore.java index 39584a48199a..e6d467cc64d1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/DynamicSemaphore.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/DynamicSemaphore.java @@ -76,6 +76,10 @@ public int availablePermits() { return semaphore.availablePermits(); } + public int getWaitingThreadCount() { + return semaphore.getQueueLength(); + } + /** * Returns the number of acquired permits. The result can exceed the configured maximum * while a reduced permit limit waits for current holders to release permits. diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java index 8d9e6dd331a3..8b4cbdc848be 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java @@ -238,6 +238,14 @@ public SchedulingAgent getSchedulingAgent(final SchedulingStrategy strategy) { return strategyAgentMap.get(strategy); } + @Override + public void notifySchedulingEvent(final Connectable connectable) { + final SchedulingAgent schedulingAgent = getSchedulingAgent(connectable); + if (schedulingAgent != null) { + schedulingAgent.onEvent(connectable); + } + } + private SchedulingAgent getSchedulingAgent(final Connectable connectable) { return getSchedulingAgent(connectable.getSchedulingStrategy()); } @@ -409,9 +417,13 @@ public Future unschedule(final ReportingTaskNode taskNode) { @Override public synchronized CompletableFuture startProcessor(final ProcessorNode procNode, final boolean failIfStopping) { final LifecycleState lifecycleState = getLifecycleState(requireNonNull(procNode), true, false); + final SchedulingAgent schedulingAgent = getSchedulingAgent(procNode); + final int processContextConcurrencyLimit = schedulingAgent.getProcessContextConcurrencyLimit(procNode); - final Supplier processContextFactory = () -> new StandardProcessContext(procNode, getControllerServiceProvider(), - getStateManager(procNode), lifecycleState::isTerminated, nodeTypeProvider); + final Supplier processContextFactory = () -> StandardProcessContext.createBuilder( + procNode, getControllerServiceProvider(), getStateManager(procNode), lifecycleState::isTerminated, nodeTypeProvider) + .setMaxConcurrentTasks(processContextConcurrencyLimit) + .build(); final boolean scheduleActions = procNode.getProcessGroup().resolveExecutionEngine() != ExecutionEngine.STATELESS; @@ -423,7 +435,7 @@ public void trigger() { // If using stateless engine, no need to schedule the component to run within the standard NiFi scheduler. if (scheduleActions) { - getSchedulingAgent(procNode).schedule(procNode, lifecycleState); + schedulingAgent.schedule(procNode, lifecycleState); } processorStartFutures.remove(procNode.getIdentifier(), future); @@ -554,8 +566,10 @@ public void trigger() { public Future runProcessorOnce(ProcessorNode procNode, final Callable> stopCallback) { final LifecycleState lifecycleState = getLifecycleState(requireNonNull(procNode), true, false); - final Supplier processContextFactory = () -> new StandardProcessContext(procNode, getControllerServiceProvider(), - getStateManager(procNode), lifecycleState::isTerminated, nodeTypeProvider); + final Supplier processContextFactory = () -> StandardProcessContext.createBuilder( + procNode, getControllerServiceProvider(), getStateManager(procNode), lifecycleState::isTerminated, nodeTypeProvider) + .setMaxConcurrentTasks(1) + .build(); final CompletableFuture future = new CompletableFuture<>(); final SchedulingAgentCallback callback = new SchedulingAgentCallback() { @@ -594,12 +608,16 @@ public void onTaskComplete() { @Override public synchronized CompletableFuture stopProcessor(final ProcessorNode procNode, final ProcessorStopLifecycleMethods lifecycleMethods) { final LifecycleState lifecycleState = getLifecycleState(procNode, false, false); + final SchedulingAgent schedulingAgent = getSchedulingAgent(procNode); + final int processContextConcurrencyLimit = schedulingAgent.getProcessContextConcurrencyLimit(procNode); - final StandardProcessContext processContext = new StandardProcessContext(procNode, getControllerServiceProvider(), - getStateManager(procNode), lifecycleState::isTerminated, nodeTypeProvider); + final StandardProcessContext processContext = StandardProcessContext.createBuilder( + procNode, getControllerServiceProvider(), getStateManager(procNode), lifecycleState::isTerminated, nodeTypeProvider) + .setMaxConcurrentTasks(processContextConcurrencyLimit) + .build(); LOG.info("Stopping {}", procNode); - final CompletableFuture stopFuture = procNode.stop(this, this.componentLifeCycleThreadPool, processContext, getSchedulingAgent(procNode), lifecycleState, lifecycleMethods); + final CompletableFuture stopFuture = procNode.stop(this, this.componentLifeCycleThreadPool, processContext, schedulingAgent, lifecycleState, lifecycleMethods); final CompletableFuture startFuture = processorStartFutures.remove(procNode.getIdentifier()); if (startFuture != null) { startFuture.completeExceptionally(new CancellationException("Processor start cancelled by stop request")); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java index 69fe3031a6ea..4edcbdb9a250 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java @@ -20,9 +20,11 @@ import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.tasks.ConnectableTask; +import org.apache.nifi.controller.tasks.InvocationObserver; import org.apache.nifi.controller.tasks.InvocationResult; import org.apache.nifi.controller.tasks.ReportingTaskWrapper; import org.apache.nifi.engine.FlowEngine; +import org.apache.nifi.scheduling.SchedulingStrategy; import org.apache.nifi.util.FormatUtils; import org.apache.nifi.util.NiFiProperties; @@ -43,7 +45,8 @@ public TimerDrivenSchedulingAgent(final FlowController flowController, final Flo try { noWorkYieldNanos = FormatUtils.getTimeDuration(boredYieldDuration, TimeUnit.NANOSECONDS); } catch (final IllegalArgumentException e) { - throw new RuntimeException("Failed to create SchedulingAgent because the " + NiFiProperties.BORED_YIELD_DURATION + " property is set to an invalid time duration: " + boredYieldDuration); + throw new IllegalStateException("Failed to create SchedulingAgent because the " + NiFiProperties.BORED_YIELD_DURATION + + " property is set to an invalid time duration: " + boredYieldDuration, e); } } @@ -69,12 +72,16 @@ public void doSchedule(final ReportingTaskNode taskNode, final LifecycleState sc public void doSchedule(final Connectable connectable, final LifecycleState scheduleState) { final List> futures = new ArrayList<>(); final ConnectableTask connectableTask = new ConnectableTask(this, connectable, flowController, contextFactory, scheduleState); + final boolean automaticScheduling = connectable.getSchedulingStrategy() == SchedulingStrategy.AUTO; + final int taskCount = automaticScheduling ? 1 : connectable.getMaxConcurrentTasks(); + final long effectiveRunDurationNanos = automaticScheduling && connectable.isSessionBatchingSupported() ? TimeUnit.MILLISECONDS.toNanos(25L) + : connectable.getRunDuration(TimeUnit.NANOSECONDS); - for (int i = 0; i < connectable.getMaxConcurrentTasks(); i++) { + for (int i = 0; i < taskCount; i++) { // Determine the task to run and create it. final AtomicReference> futureRef = new AtomicReference<>(); - final Runnable trigger = createTrigger(connectableTask, scheduleState, futureRef); + final Runnable trigger = createTrigger(connectableTask, scheduleState, futureRef, effectiveRunDurationNanos); // Schedule the task to run final ScheduledFuture future = flowEngine.scheduleWithFixedDelay(trigger, 0L, @@ -89,17 +96,18 @@ public void doSchedule(final Connectable connectable, final LifecycleState sched } scheduleState.setFutures(futures); - logger.info("Scheduled {} to run with {} threads", connectable, connectable.getMaxConcurrentTasks()); + logger.info("Scheduled {} to run with {} threads", connectable, taskCount); } - private Runnable createTrigger(final ConnectableTask connectableTask, final LifecycleState scheduleState, final AtomicReference> futureRef) { + private Runnable createTrigger(final ConnectableTask connectableTask, final LifecycleState scheduleState, final AtomicReference> futureRef, + final long effectiveRunDurationNanos) { final Connectable connectable = connectableTask.getConnectable(); final Runnable yieldDetectionRunnable = new Runnable() { @Override public void run() { // Call the task. It will return a boolean indicating whether or not we should yield // based on a lack of work for to do for the component. - final InvocationResult invocationResult = connectableTask.invoke(); + final InvocationResult invocationResult = connectableTask.invoke(effectiveRunDurationNanos, scheduleState::isScheduled, InvocationObserver.NO_OP); if (invocationResult.isYield()) { logger.debug("Yielding {} due to {}", connectable, invocationResult.getYieldExplanation()); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java index 911b700a0c35..dee3e2c2cbfb 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java @@ -17,21 +17,45 @@ package org.apache.nifi.controller.scheduling; import org.apache.nifi.connectable.Connectable; +import org.apache.nifi.connectable.Connection; import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.Triggerable; +import org.apache.nifi.controller.queue.FlowFileQueue; +import org.apache.nifi.controller.queue.QueueSchedulingRegistration; +import org.apache.nifi.controller.scheduling.auto.AutoSchedulingDiagnostics; +import org.apache.nifi.controller.scheduling.auto.AutoSchedulingResetReason; +import org.apache.nifi.controller.scheduling.auto.ConcurrencyEvaluationState; +import org.apache.nifi.controller.scheduling.auto.ConcurrencyUpdateStatus; +import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingDecision; +import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingDecisionReason; +import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingMeasurements; +import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingSnapshot; +import org.apache.nifi.controller.scheduling.auto.StandardAutoSchedulingController; +import org.apache.nifi.controller.scheduling.auto.SystemSchedulingMetrics; +import org.apache.nifi.controller.scheduling.auto.SystemSchedulingSnapshot; import org.apache.nifi.controller.tasks.ConnectableTask; +import org.apache.nifi.controller.tasks.InvocationObserver; +import org.apache.nifi.controller.tasks.InvocationOutcome; import org.apache.nifi.controller.tasks.InvocationResult; import org.apache.nifi.controller.tasks.ReportingTaskWrapper; import org.apache.nifi.nar.NarThreadContextClassLoader; import org.apache.nifi.scheduling.SchedulingStrategy; +import org.apache.nifi.util.Connectables; import org.apache.nifi.util.FormatUtils; import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.support.CronExpression; +import java.time.Duration; +import java.time.Instant; import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -40,10 +64,16 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.LongAdder; +import java.util.concurrent.locks.LockSupport; /** * Scheduling agent that runs components on virtual threads. A {@link DynamicSemaphore} @@ -58,19 +88,50 @@ public class VirtualThreadSchedulingAgent implements SchedulingAgent { private final FlowController flowController; private final RepositoryContextFactory contextFactory; private final DynamicSemaphore globalSemaphore; + private final int autoMaxConcurrentTasks; + private final int availableProcessorCount; + private final boolean autoMaxConcurrentTasksBasedOnAvailableProcessors; private final long noWorkYieldNanos; private final ExecutorService executorService; + private final ScheduledExecutorService schedulingEvaluationExecutor; + private final SystemSchedulingMetrics systemSchedulingMetrics; private final ConcurrentMap schedulingGenerations = new ConcurrentHashMap<>(); + private final LongAdder globalPermitHoldNanos = new LongAdder(); + private final LongAdder globalPermitWaitNanos = new LongAdder(); + private final LongAdder globalTaskSlotsFullyUsedSamples = new LongAdder(); + private final LongAdder globalTaskUsageSamples = new LongAdder(); private final AtomicBoolean shutdown = new AtomicBoolean(); private final AtomicInteger runningThreadCount = new AtomicInteger(); + private volatile SystemSchedulingSnapshot systemSchedulingSnapshot = SystemSchedulingSnapshot.createBuilder() + .setCpuLoad(-1D) + .setAverageCpuLoad(-1D) + .setMaxGlobalConcurrentTasks(1) + .build(); + private volatile long lastSystemSchedulingSnapshotNanos = System.nanoTime(); private volatile String adminYieldDuration = "1 sec"; private volatile long adminYieldNanos = TimeUnit.SECONDS.toNanos(1L); public VirtualThreadSchedulingAgent(final FlowController flowController, final RepositoryContextFactory contextFactory, final NiFiProperties nifiProperties, final int maxThreadCount) { + this(flowController, contextFactory, nifiProperties, maxThreadCount, nifiProperties.getProcessorAutoMaxConcurrentTasks()); + } + + public VirtualThreadSchedulingAgent(final FlowController flowController, final RepositoryContextFactory contextFactory, + final NiFiProperties nifiProperties, final int maxThreadCount, final int autoMaxConcurrentTasks) { + this(flowController, contextFactory, nifiProperties, maxThreadCount, autoMaxConcurrentTasks, new SystemSchedulingMetrics()); + } + + VirtualThreadSchedulingAgent(final FlowController flowController, final RepositoryContextFactory contextFactory, final NiFiProperties nifiProperties, + final int maxThreadCount, final int autoMaxConcurrentTasks, final SystemSchedulingMetrics systemSchedulingMetrics) { + this.systemSchedulingMetrics = systemSchedulingMetrics; this.flowController = flowController; this.contextFactory = contextFactory; this.globalSemaphore = new DynamicSemaphore(maxThreadCount); + this.autoMaxConcurrentTasks = autoMaxConcurrentTasks; + availableProcessorCount = Runtime.getRuntime().availableProcessors(); + final String configuredMaximum = nifiProperties.getProperty(NiFiProperties.PROCESSOR_AUTO_MAX_CONCURRENT_TASKS); + final int configuredMaxConcurrentTasks = configuredMaximum == null ? autoMaxConcurrentTasks : Integer.parseInt(configuredMaximum.trim()); + autoMaxConcurrentTasksBasedOnAvailableProcessors = configuredMaxConcurrentTasks > 4L * availableProcessorCount; final String boredYieldDuration = nifiProperties.getBoredYieldDuration(); try { @@ -86,17 +147,34 @@ public VirtualThreadSchedulingAgent(final FlowController flowController, final R return thread; }; executorService = Executors.newThreadPerTaskExecutor(threadFactory); + schedulingEvaluationExecutor = Executors.newSingleThreadScheduledExecutor(runnable -> { + final Thread thread = Thread.ofPlatform().daemon(true).name("Automatic Processor Scheduling Controller").unstarted(runnable); + thread.setContextClassLoader(NarThreadContextClassLoader.getInstance()); + return thread; + }); + schedulingEvaluationExecutor.scheduleWithFixedDelay(this::evaluateAutoScheduling, 250L, 250L, TimeUnit.MILLISECONDS); logger.info("VirtualThreadSchedulingAgent initialized with {} permits", maxThreadCount); } + @Override + public int getProcessContextConcurrencyLimit(final Connectable connectable) { + if (connectable.getSchedulingStrategy() != SchedulingStrategy.AUTO) { + return connectable.getMaxConcurrentTasks(); + } + + return connectable instanceof ProcessorNode processorNode && processorNode.isTriggeredSerially() ? 1 : autoMaxConcurrentTasks; + } + @Override public void shutdown() { signalShutdown(true); + schedulingEvaluationExecutor.shutdownNow(); executorService.shutdownNow(); } public void shutdownGracefully() { signalShutdown(false); + schedulingEvaluationExecutor.shutdown(); executorService.shutdown(); } @@ -109,11 +187,17 @@ private void signalShutdown(final boolean interrupt) { } public boolean awaitTermination(final long timeout, final TimeUnit timeUnit) throws InterruptedException { - return executorService.awaitTermination(timeout, timeUnit); + final long deadlineNanos = System.nanoTime() + timeUnit.toNanos(timeout); + if (!schedulingEvaluationExecutor.awaitTermination(timeout, timeUnit)) { + return false; + } + + final long remainingNanos = Math.max(0L, deadlineNanos - System.nanoTime()); + return executorService.awaitTermination(remainingNanos, TimeUnit.NANOSECONDS); } public boolean isTerminated() { - return executorService.isTerminated(); + return schedulingEvaluationExecutor.isTerminated() && executorService.isTerminated(); } @Override @@ -139,8 +223,19 @@ public void schedule(final Connectable connectable, final LifecycleState lifecyc try { final ConnectableTask connectableTask = new ConnectableTask(this, connectable, flowController, contextFactory, lifecycleState, generation::isRunning); - final int taskCount = connectable.getMaxConcurrentTasks(); + if (connectable.getSchedulingStrategy() == SchedulingStrategy.AUTO) { + final int maxConcurrentTasks = getProcessContextConcurrencyLimit(connectable); + final ProcessorAutoSchedulingState autoSchedulingState = new ProcessorAutoSchedulingState(connectable, connectableTask, lifecycleState, generation, + maxConcurrentTasks, connectable.isSessionBatchingSupported(), + connectable instanceof ProcessorNode processorNode && processorNode.isTriggeredSerially()); + generation.setAutoSchedulingState(autoSchedulingState); + registerQueueListeners(connectable, generation); + startAutoWorkers(autoSchedulingState, 1); + logger.info("Scheduled {} for automatic scheduling with a maximum of {} concurrent tasks", connectable, maxConcurrentTasks); + return; + } + final int taskCount = connectable.getMaxConcurrentTasks(); for (int i = 0; i < taskCount; i++) { final String threadName = buildThreadName(connectable, i); submitTask(threadName, generation, () -> runSchedulingLoop(connectable, connectableTask, schedulingNanos, lifecycleState, generation, cronExpression)); @@ -158,6 +253,119 @@ public void schedule(final Connectable connectable, final LifecycleState lifecyc } } + private void registerQueueListeners(final Connectable connectable, final SchedulingGeneration generation) { + final Set queues = new HashSet<>(); + for (final Connection connection : connectable.getIncomingConnections()) { + queues.add(connection.getFlowFileQueue()); + } + + for (final Connection connection : connectable.getConnections()) { + queues.add(connection.getFlowFileQueue()); + } + + for (final FlowFileQueue queue : queues) { + generation.addQueue(queue); + generation.addQueueRegistration(queue.addSchedulingListener(generation::signalChange)); + } + } + + private void startAutoWorkers(final ProcessorAutoSchedulingState state, final int desiredWorkerCount) { + final List workers = state.resizeWorkers(desiredWorkerCount); + for (final ProcessorTaskWorker worker : workers) { + final String threadName = buildThreadName(state.connectable, Math.toIntExact(worker.identifier)); + submitTask(threadName, state.generation, () -> runAutoSchedulingLoop(state, worker)); + } + } + + private void runAutoSchedulingLoop(final ProcessorAutoSchedulingState state, final ProcessorTaskWorker worker) { + try { + while (isActive(state.lifecycleState, state.generation) && !worker.retired.get()) { + if (!state.connectableTask.isReady() || !state.isSourceWorkCheckAllowed()) { + waitForAutoReadiness(state, worker); + continue; + } + + final long concurrentTaskSlotChangeCount = state.generation.getConcurrentTaskSlotChangeCount(); + if (!acquirePermitWithPolling(state.lifecycleState, state.generation)) { + return; + } + + final SchedulingSettings schedulingSettings = state.schedulingSettings.get(); + final long permitHoldStartNanos = System.nanoTime(); + boolean sourceWorkCheckReserved = false; + boolean concurrentTaskSlotAcquired = false; + boolean invocationAttempted = false; + boolean concurrentTaskSlotUnavailable = false; + try { + if (!isActive(state.lifecycleState, state.generation) || worker.retired.get() || !state.connectableTask.isReady()) { + continue; + } + + concurrentTaskSlotAcquired = state.tryAcquireConcurrentTaskSlot(schedulingSettings); + if (concurrentTaskSlotAcquired) { + sourceWorkCheckReserved = state.reserveSourceWorkCheck(); + if (state.requiresSourceWorkCheck() && !sourceWorkCheckReserved) { + continue; + } + + invocationAttempted = true; + final InvocationObserver observer = state.measurements.createInvocationObserver(); + final InvocationResult result = state.connectableTask.invoke(schedulingSettings.runDurationNanos(), + () -> isActive(state.lifecycleState, state.generation) && !worker.retired.get() + && state.schedulingSettings.get() == schedulingSettings, observer); + state.recordInvocationResult(result); + } else { + concurrentTaskSlotUnavailable = true; + } + } finally { + if (concurrentTaskSlotAcquired) { + state.releaseConcurrentTaskSlot(); + } + + if (sourceWorkCheckReserved) { + state.releaseSourceWorkCheck(); + } + + final long permitHoldNanos = System.nanoTime() - permitHoldStartNanos; + if (invocationAttempted) { + state.measurements.recordInvocationDuration(permitHoldNanos); + } + + globalPermitHoldNanos.add(permitHoldNanos); + Thread.interrupted(); + globalSemaphore.release(); + } + + if (concurrentTaskSlotUnavailable) { + state.generation.waitForConcurrentTaskSlot(concurrentTaskSlotChangeCount, worker); + } + } + } finally { + state.workerStopped(worker); + } + } + + private void waitForAutoReadiness(final ProcessorAutoSchedulingState state, final ProcessorTaskWorker worker) { + final long changeSequence = state.generation.getChangeSequence(); + if (state.connectableTask.isReady() && state.isSourceWorkCheckAllowed()) { + return; + } + + final Instant now = Instant.now(); + long delayNanos = TimeUnit.SECONDS.toNanos(1L); + final Instant queueDeadline = state.getNextQueueDeadline(); + if (queueDeadline.isAfter(now)) { + delayNanos = Math.min(delayNanos, Duration.between(now, queueDeadline).toNanos()); + } + + final long sourceDelayNanos = state.getSourceWorkCheckDelayNanos(); + if (sourceDelayNanos > 0L) { + delayNanos = Math.min(delayNanos, sourceDelayNanos); + } + + state.generation.awaitChange(changeSequence, delayNanos, worker); + } + @Override public void scheduleOnce(final Connectable connectable, final LifecycleState lifecycleState, final Callable> stopCallback) { final String componentId = connectable.getIdentifier(); @@ -285,6 +493,65 @@ private boolean isActive(final LifecycleState lifecycleState, final SchedulingGe return !shutdown.get() && lifecycleState.isScheduled() && !generation.isStopped(); } + private void evaluateAutoScheduling() { + try { + final long nowNanos = System.nanoTime(); + final int maxGlobalConcurrentTasks = globalSemaphore.getMaxPermits(); + globalTaskUsageSamples.increment(); + if (globalSemaphore.getInUsePermits() >= maxGlobalConcurrentTasks) { + globalTaskSlotsFullyUsedSamples.increment(); + } + + if (nowNanos - lastSystemSchedulingSnapshotNanos >= TimeUnit.SECONDS.toNanos(1L)) { + final long taskUsageSamples = globalTaskUsageSamples.sumThenReset(); + final double fractionOfSamplesWithAllGlobalTaskSlotsUsed = taskUsageSamples == 0L + ? 0D : globalTaskSlotsFullyUsedSamples.sumThenReset() / (double) taskUsageSamples; + final long permitWaitNanos = globalPermitWaitNanos.sumThenReset(); + final long permitHoldNanos = globalPermitHoldNanos.sumThenReset(); + final double permitUseNanos = (double) permitWaitNanos + permitHoldNanos; + final double fractionOfGlobalTaskTimeSpentWaiting = permitUseNanos == 0D ? 0D : permitWaitNanos / permitUseNanos; + systemSchedulingSnapshot = systemSchedulingMetrics.captureSnapshot(maxGlobalConcurrentTasks, + fractionOfSamplesWithAllGlobalTaskSlotsUsed, fractionOfGlobalTaskTimeSpentWaiting); + lastSystemSchedulingSnapshotNanos = nowNanos; + } + + for (final SchedulingGeneration generation : schedulingGenerations.values()) { + final ProcessorAutoSchedulingState state = generation.getAutoSchedulingState(); + if (state == null || generation.isStopped()) { + continue; + } + + state.recordReadiness(nowNanos); + startAutoWorkers(state, state.schedulingSettings.get().concurrentTasks()); + if (!state.isEvaluationDue(nowNanos)) { + continue; + } + + final ProcessorSchedulingSnapshot processorMeasurements = state.captureProcessorSchedulingSnapshot(nowNanos); + final long resetSequence = state.resetSequence.get(); + final ProcessorSchedulingDecision decision = state.controller.evaluate(processorMeasurements, systemSchedulingSnapshot); + logger.debug("Automatic scheduling evaluation for {}: processorMeasurements={}, systemMeasurements={}, decision={}", + state.connectable, processorMeasurements, systemSchedulingSnapshot, decision); + if (generation.isStopped() || schedulingGenerations.get(state.connectable.getIdentifier()) != generation + || state.resetSequence.get() != resetSequence) { + continue; + } + + state.recordDecision(decision); + if (decision.applySettings()) { + final int selectedConcurrentTasks = Math.max(1, Math.min(decision.settings().concurrentTasks(), + Math.min(state.maxConcurrentTasks, globalSemaphore.getMaxPermits()))); + final SchedulingSettings settings = new SchedulingSettings(selectedConcurrentTasks, + state.batchingSupported ? Math.min(decision.settings().runDurationNanos(), TimeUnit.MILLISECONDS.toNanos(25L)) : 0L); + state.applySettings(settings); + startAutoWorkers(state, selectedConcurrentTasks); + } + } + } catch (final Throwable failure) { + logger.error("Failed to evaluate automatic Processor scheduling", failure); + } + } + private static CronExpression parseCronExpression(final String cronSchedule, final Object component) { try { return CronExpression.parse(cronSchedule); @@ -295,11 +562,21 @@ private static CronExpression parseCronExpression(final String cronSchedule, fin @Override public void onEvent(final Connectable connectable) { + final SchedulingGeneration generation = schedulingGenerations.get(connectable.getIdentifier()); + if (generation != null) { + final ProcessorAutoSchedulingState state = generation.getAutoSchedulingState(); + if (state != null) { + requestControllerReset(generation, state, AutoSchedulingResetReason.PROCESSOR_CONNECTIONS_CHANGED); + } + + generation.signalChange(); + } } @Override public synchronized void setMaxThreadCount(final int maxThreads) { globalSemaphore.setMaxPermits(maxThreads); + resetAutoControllersForGlobalConcurrentTaskLimitChange(); logger.info("Global semaphore permits updated to {}", maxThreads); } @@ -316,6 +593,52 @@ public synchronized void incrementMaxThreadCount(final int toAdd) { } globalSemaphore.setMaxPermits(newMax); + resetAutoControllersForGlobalConcurrentTaskLimitChange(); + } + + private void resetAutoControllersForGlobalConcurrentTaskLimitChange() { + for (final SchedulingGeneration generation : schedulingGenerations.values()) { + final ProcessorAutoSchedulingState state = generation.getAutoSchedulingState(); + if (state != null) { + requestControllerReset(generation, state, AutoSchedulingResetReason.GLOBAL_CONCURRENT_TASK_LIMIT_CHANGED); + generation.signalChange(); + } + } + } + + private void requestControllerReset(final SchedulingGeneration generation, final ProcessorAutoSchedulingState state, + final AutoSchedulingResetReason reason) { + final long resetSequence = state.resetSequence.incrementAndGet(); + try { + schedulingEvaluationExecutor.execute(() -> { + if (generation.isStopped() || state.resetSequence.get() != resetSequence) { + return; + } + + if (reason == AutoSchedulingResetReason.PROCESSOR_CONNECTIONS_CHANGED) { + generation.clearQueueRegistrations(); + registerQueueListeners(state.connectable, generation); + } + + state.resetMeasurementsAfterEvent(); + if (reason == AutoSchedulingResetReason.GLOBAL_CONCURRENT_TASK_LIMIT_CHANGED) { + final SchedulingSettings currentSettings = state.schedulingSettings.get(); + final int selectedConcurrentTasks = Math.min(currentSettings.concurrentTasks(), globalSemaphore.getMaxPermits()); + final SchedulingSettings clampedSettings = new SchedulingSettings(selectedConcurrentTasks, currentSettings.runDurationNanos()); + state.applySettings(clampedSettings); + startAutoWorkers(state, selectedConcurrentTasks); + } + + state.controller.reset(); + state.lastDecision = new ProcessorSchedulingDecision(state.schedulingSettings.get(), + ConcurrencyEvaluationState.USING_CURRENT_CONCURRENCY, + ProcessorSchedulingDecisionReason.PERFORMANCE_MEASUREMENTS_RESET, false); + }); + } catch (final RejectedExecutionException e) { + if (!shutdown.get()) { + throw e; + } + } } @Override @@ -342,6 +665,38 @@ int getRunningThreadCount() { return runningThreadCount.get(); } + void requestAutoSchedulingSettings(final String componentIdentifier, final SchedulingSettings settings) { + final SchedulingGeneration generation = schedulingGenerations.get(componentIdentifier); + if (generation == null || generation.getAutoSchedulingState() == null) { + throw new IllegalStateException("Component " + componentIdentifier + " is not scheduled using automatic scheduling"); + } + + schedulingEvaluationExecutor.execute(() -> { + final ProcessorAutoSchedulingState state = generation.getAutoSchedulingState(); + if (state == null || generation.isStopped() || schedulingGenerations.get(componentIdentifier) != generation) { + return; + } + + state.applySettings(settings); + startAutoWorkers(state, settings.concurrentTasks()); + }); + } + + int getAutoSchedulingWaiterCount(final String componentIdentifier) { + final SchedulingGeneration generation = schedulingGenerations.get(componentIdentifier); + return generation == null ? 0 : generation.getWaiterCount(); + } + + int getAutoSchedulingConcurrentTaskSlotWaiterCount(final String componentIdentifier) { + final SchedulingGeneration generation = schedulingGenerations.get(componentIdentifier); + return generation == null ? 0 : generation.getConcurrentTaskSlotWaiterCount(); + } + + long getAutoSchedulingConcurrentTaskSlotChangeCount(final String componentIdentifier) { + final SchedulingGeneration generation = schedulingGenerations.get(componentIdentifier); + return generation == null ? -1L : generation.getConcurrentTaskSlotChangeCount(); + } + boolean isShutdown() { return shutdown.get(); } @@ -353,6 +708,12 @@ public int getActiveThreadCount() { return globalSemaphore.getInUsePermits(); } + public AutoSchedulingDiagnostics getAutoSchedulingDiagnostics(final String componentIdentifier) { + final SchedulingGeneration generation = schedulingGenerations.get(componentIdentifier); + final ProcessorAutoSchedulingState state = generation == null ? null : generation.getAutoSchedulingState(); + return state == null ? null : state.getDiagnostics(); + } + private void runSchedulingLoop(final Connectable connectable, final ConnectableTask connectableTask, final long schedulingNanos, final LifecycleState lifecycleState, final SchedulingGeneration generation, final CronExpression cronExpression) { final boolean cronDriven = cronExpression != null; @@ -378,11 +739,13 @@ private void runSchedulingLoop(final Connectable connectable, final ConnectableT } final InvocationResult invocationResult; + final long permitHoldStartNanos = System.nanoTime(); try { invocationResult = connectableTask.invoke(); } finally { // Interrupt status from one invocation must not carry into the scheduling loop. Thread.interrupted(); + globalPermitHoldNanos.add(System.nanoTime() - permitHoldStartNanos); globalSemaphore.release(); } @@ -428,9 +791,11 @@ private void runOnce(final Connectable connectable, final ConnectableTask connec return; } + final long permitHoldStartNanos = System.nanoTime(); try { connectableTask.invoke(); } finally { + globalPermitHoldNanos.add(System.nanoTime() - permitHoldStartNanos); globalSemaphore.release(); } } catch (final Throwable t) { @@ -469,11 +834,13 @@ private void runReportingTaskLoop(final ReportingTaskNode taskNode, final Runnab return; } + final long permitHoldStartNanos = System.nanoTime(); try { reportingTaskWrapper.run(); } finally { // Interrupt status from one invocation must not carry into the scheduling loop. Thread.interrupted(); + globalPermitHoldNanos.add(System.nanoTime() - permitHoldStartNanos); globalSemaphore.release(); } @@ -524,10 +891,12 @@ private void waitForNextInvocation(final Connectable connectable, final long sch } private boolean acquirePermitWithPolling(final LifecycleState lifecycleState, final SchedulingGeneration generation) { + final long waitStartNanos = System.nanoTime(); while (isActive(lifecycleState, generation)) { try { if (globalSemaphore.tryAcquire(PERMIT_POLL_INTERVAL_NANOS, TimeUnit.NANOSECONDS)) { if (isActive(lifecycleState, generation)) { + globalPermitWaitNanos.add(System.nanoTime() - waitStartNanos); return true; } @@ -585,10 +954,415 @@ private static OffsetDateTime getNextCronSchedule(final OffsetDateTime currentSc return cronExpression.next(now.isAfter(currentSchedule) ? now : currentSchedule); } - private static class SchedulingGeneration { + private static class ProcessorTaskWorker { + private final long identifier; + private final AtomicBoolean retired = new AtomicBoolean(); + + private ProcessorTaskWorker(final long identifier) { + this.identifier = identifier; + } + } + + private record WaitingProcessorTask(Thread thread, ProcessorTaskWorker worker) { + } + + private class ProcessorAutoSchedulingState { + private final Connectable connectable; + private final ConnectableTask connectableTask; + private final LifecycleState lifecycleState; + private final SchedulingGeneration generation; + private final int maxConcurrentTasks; + private final boolean batchingSupported; + private final boolean sourceProcessor; + private final StandardAutoSchedulingController controller; + private final ProcessorSchedulingMeasurements measurements = new ProcessorSchedulingMeasurements(); + private final AtomicReference schedulingSettings; + private final AtomicInteger activeProcessorInvocations = new AtomicInteger(); + private final ConcurrentMap workers = new ConcurrentHashMap<>(); + private final AtomicLong workerSequence = new AtomicLong(); + private final AtomicBoolean sourceWorkCheckInProgress = new AtomicBoolean(); + private final AtomicLong resetSequence = new AtomicLong(); + private final long evaluationIntervalNanos; + private final long baseSourceDelayNanos; + private final long maximumSourceDelayNanos; + + private volatile long nextEvaluationNanos; + private volatile long lastEvaluationNanos = System.nanoTime(); + private volatile long lastReadinessMeasurementNanos = System.nanoTime(); + private volatile long previousInputQueueSize; + private volatile long sourceDelayNanos; + private volatile long nextSourceWorkCheckNanos; + private volatile long lastSourceActivityNanos; + private volatile ProcessorSchedulingDecision lastDecision; + private volatile ProcessorSchedulingDecision lastChangeDecision; + private volatile ProcessorSchedulingSnapshot lastProcessorSchedulingSnapshot; + private volatile boolean previousPrimaryNode; + private volatile boolean measurementsSupported; + + private ProcessorAutoSchedulingState(final Connectable connectable, final ConnectableTask connectableTask, + final LifecycleState lifecycleState, final SchedulingGeneration generation, + final int maxConcurrentTasks, final boolean batchingSupported, + final boolean processorTriggeredSerially) { + this.connectable = connectable; + this.connectableTask = connectableTask; + this.lifecycleState = lifecycleState; + this.generation = generation; + this.maxConcurrentTasks = maxConcurrentTasks; + this.batchingSupported = batchingSupported; + schedulingSettings = new AtomicReference<>(new SchedulingSettings(1, batchingSupported ? TimeUnit.MILLISECONDS.toNanos(25L) : 0L)); + sourceProcessor = connectable.isTriggerWhenEmpty() || !connectable.hasIncomingConnection() || !Connectables.hasNonLoopConnection(connectable); + controller = new StandardAutoSchedulingController(maxConcurrentTasks, batchingSupported, processorTriggeredSerially, + autoMaxConcurrentTasksBasedOnAvailableProcessors, availableProcessorCount); + final double stagger = ((Math.floorMod(connectable.getIdentifier().hashCode(), 201) - 100) / 1000D); + this.evaluationIntervalNanos = (long) (TimeUnit.SECONDS.toNanos(1L) * (1D + stagger)); + this.nextEvaluationNanos = lastEvaluationNanos + evaluationIntervalNanos; + this.baseSourceDelayNanos = Math.max(noWorkYieldNanos, TimeUnit.MILLISECONDS.toNanos(1L)); + this.maximumSourceDelayNanos = Math.max(baseSourceDelayNanos, TimeUnit.MILLISECONDS.toNanos(100L)); + this.previousPrimaryNode = flowController.isPrimary(); + } + + synchronized List resizeWorkers(final int targetWorkerCount) { + final List activeWorkers = new ArrayList<>(); + for (final ProcessorTaskWorker worker : workers.values()) { + if (!worker.retired.get()) { + activeWorkers.add(worker); + } + } + + if (activeWorkers.size() > targetWorkerCount) { + activeWorkers.sort(Comparator.comparingLong(worker -> worker.identifier)); + for (int index = targetWorkerCount; index < activeWorkers.size(); index++) { + activeWorkers.get(index).retired.set(true); + } + + generation.signalChange(); + generation.signalConcurrentTaskSlotChange(); + return List.of(); + } + + final List newWorkers = new ArrayList<>(); + for (int index = activeWorkers.size(); index < targetWorkerCount; index++) { + final ProcessorTaskWorker worker = new ProcessorTaskWorker(workerSequence.getAndIncrement()); + workers.put(worker.identifier, worker); + newWorkers.add(worker); + } + + return newWorkers; + } + + void workerStopped(final ProcessorTaskWorker worker) { + workers.remove(worker.identifier, worker); + generation.signalChange(); + generation.signalConcurrentTaskSlotChange(); + } + + boolean tryAcquireConcurrentTaskSlot(final SchedulingSettings expectedSettings) { + while (true) { + if (schedulingSettings.get() != expectedSettings) { + return false; + } + + final int currentActiveProcessorInvocations = activeProcessorInvocations.get(); + if (currentActiveProcessorInvocations >= expectedSettings.concurrentTasks()) { + return false; + } + + if (activeProcessorInvocations.compareAndSet(currentActiveProcessorInvocations, currentActiveProcessorInvocations + 1)) { + if (schedulingSettings.get() == expectedSettings) { + measurements.recordInvocationStarted(); + return true; + } + + activeProcessorInvocations.decrementAndGet(); + generation.signalConcurrentTaskSlotChange(); + return false; + } + } + } + + void releaseConcurrentTaskSlot() { + measurements.recordInvocationFinished(); + activeProcessorInvocations.decrementAndGet(); + generation.signalConcurrentTaskSlotChange(); + } + + boolean isSourceWorkCheckAllowed() { + return !sourceProcessor || connectableTask.hasLocallyConsumableInput() || sourceDelayNanos == 0L + || (System.nanoTime() >= nextSourceWorkCheckNanos && !sourceWorkCheckInProgress.get()); + } + + boolean reserveSourceWorkCheck() { + return requiresSourceWorkCheck() && sourceWorkCheckInProgress.compareAndSet(false, true); + } + + boolean requiresSourceWorkCheck() { + return sourceProcessor && !connectableTask.hasLocallyConsumableInput() && sourceDelayNanos > 0L; + } + + void releaseSourceWorkCheck() { + sourceWorkCheckInProgress.set(false); + generation.signalChange(); + } + + long getSourceWorkCheckDelayNanos() { + return Math.max(0L, nextSourceWorkCheckNanos - System.nanoTime()); + } + + void recordInvocationResult(final InvocationResult result) { + if (!sourceProcessor) { + return; + } + + if (result.getOutcome() == InvocationOutcome.INVOKED_WITH_ACTIVITY) { + sourceDelayNanos = 0L; + nextSourceWorkCheckNanos = 0L; + lastSourceActivityNanos = System.nanoTime(); + generation.signalChange(); + } else if (result.getOutcome() == InvocationOutcome.INVOKED_WITHOUT_ACTIVITY + && activeProcessorInvocations.get() <= 1 + && !connectableTask.hasLocallyConsumableInput()) { + sourceDelayNanos = sourceDelayNanos == 0L ? baseSourceDelayNanos : Math.min(maximumSourceDelayNanos, sourceDelayNanos * 2L); + nextSourceWorkCheckNanos = System.nanoTime() + sourceDelayNanos; + } + } + + Instant getNextQueueDeadline() { + Instant earliestDeadline = Instant.EPOCH; + for (final FlowFileQueue queue : generation.queues) { + final Instant deadline = queue.getNextFlowFileAvailabilityTime(); + if (deadline.isAfter(Instant.EPOCH) && (earliestDeadline.equals(Instant.EPOCH) || deadline.isBefore(earliestDeadline))) { + earliestDeadline = deadline; + } + } + + return earliestDeadline; + } + + boolean isEvaluationDue(final long nowNanos) { + if (nowNanos < nextEvaluationNanos) { + return false; + } + + nextEvaluationNanos = nowNanos + evaluationIntervalNanos; + return true; + } + + void recordReadiness(final long nowNanos) { + final long measurementNanos = Math.max(1L, nowNanos - lastReadinessMeasurementNanos); + lastReadinessMeasurementNanos = nowNanos; + measurements.recordReadiness(connectableTask.getReadinessOutcome(), measurementNanos); + } + + ProcessorSchedulingSnapshot captureProcessorSchedulingSnapshot(final long nowNanos) { + final SchedulingSettings currentSettings = schedulingSettings.get(); + long inputQueueSize = 0L; + for (final Connection connection : connectable.getIncomingConnections()) { + inputQueueSize += connection.getFlowFileQueue().getLocalQueueSize().getObjectCount(); + } + + final boolean inputQueueHasFlowFiles = connectableTask.hasLocallyConsumableInput(); + final double inputQueueGrowth = inputQueueSize - previousInputQueueSize; + previousInputQueueSize = inputQueueSize; + final long measurementWindowNanos = Math.max(1L, nowNanos - lastEvaluationNanos); + lastEvaluationNanos = nowNanos; + final boolean currentPrimaryNode = flowController.isPrimary(); + final boolean primaryNodeChanged = currentPrimaryNode != previousPrimaryNode; + previousPrimaryNode = currentPrimaryNode; + final boolean sourceProcessorRecentlyReportedActivity = sourceProcessor && sourceDelayNanos == 0L + && (activeProcessorInvocations.get() > 0 + || lastSourceActivityNanos > 0L && nowNanos - lastSourceActivityNanos < TimeUnit.SECONDS.toNanos(2)); + final ProcessorSchedulingSnapshot snapshot = measurements.captureSnapshot(nowNanos, measurementWindowNanos, + currentSettings, connectableTask.isReady(), inputQueueHasFlowFiles, + inputQueueGrowth, sourceProcessorRecentlyReportedActivity, primaryNodeChanged); + if (snapshot.committedFlowFiles() > 0L) { + measurementsSupported = true; + } + + lastProcessorSchedulingSnapshot = snapshot; + return snapshot; + } + + void resetMeasurementsAfterEvent() { + final long nowNanos = System.nanoTime(); + final SchedulingSettings currentSettings = schedulingSettings.get(); + // Workers compare settings by identity so a reset also ends batches whose values have not changed. + schedulingSettings.set(new SchedulingSettings(currentSettings.concurrentTasks(), currentSettings.runDurationNanos())); + lastEvaluationNanos = nowNanos; + lastReadinessMeasurementNanos = nowNanos; + long inputQueueSize = 0L; + for (final Connection connection : connectable.getIncomingConnections()) { + inputQueueSize += connection.getFlowFileQueue().getLocalQueueSize().getObjectCount(); + } + + previousInputQueueSize = inputQueueSize; + lastProcessorSchedulingSnapshot = null; + generation.signalChange(); + } + + void applySettings(final SchedulingSettings settings) { + final SchedulingSettings currentSettings = schedulingSettings.get(); + if (!currentSettings.equals(settings)) { + schedulingSettings.set(new SchedulingSettings(settings.concurrentTasks(), settings.runDurationNanos())); + generation.signalChange(); + generation.signalConcurrentTaskSlotChange(); + logger.info("Automatic scheduling settings changed for {} from {} to {}", connectable, currentSettings, settings); + } + } + + void recordDecision(final ProcessorSchedulingDecision decision) { + lastDecision = decision; + final ProcessorSchedulingDecisionReason reason = decision.reason(); + if (reason == ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_INCREASED_THROUGHPUT + || reason == ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_DID_NOT_INCREASE_THROUGHPUT + || reason == ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_MAINTAINED_THROUGHPUT + || reason == ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_REDUCED_THROUGHPUT_OR_INCREASED_INPUT_QUEUE + || reason == ProcessorSchedulingDecisionReason.COMPARISON_EXPIRED_WITHOUT_ENOUGH_MEASUREMENTS + || reason == ProcessorSchedulingDecisionReason.PERFORMANCE_MEASUREMENTS_RESET) { + lastChangeDecision = decision; + } + } + + AutoSchedulingDiagnostics getDiagnostics() { + final SchedulingSettings currentSettings = schedulingSettings.get(); + final ProcessorSchedulingDecision currentDecision = lastDecision; + final ProcessorSchedulingDecision currentChangeDecision = lastChangeDecision; + final ProcessorSchedulingSnapshot currentSnapshot = lastProcessorSchedulingSnapshot; + final long committedFlowFiles = currentSnapshot == null ? 0L : currentSnapshot.committedFlowFiles(); + final long measurementWindowNanos = currentSnapshot == null ? 0L : currentSnapshot.measurementWindowNanos(); + final double throughput = measurementWindowNanos == 0L ? 0D + : committedFlowFiles / (measurementWindowNanos / (double) TimeUnit.SECONDS.toNanos(1L)); + final String evaluationState = currentDecision == null ? ConcurrencyEvaluationState.USING_CURRENT_CONCURRENCY.name() + : currentDecision.state().name(); + final ProcessorSchedulingDecisionReason reason = currentDecision == null + ? ProcessorSchedulingDecisionReason.COLLECTING_COMPARISON_MEASUREMENTS : currentDecision.reason(); + final ConcurrencyUpdateStatus concurrencyUpdateStatus = controller.getConcurrencyUpdateStatus(); + final String lastChangeReason = currentChangeDecision == null ? null : currentChangeDecision.reason().name(); + return AutoSchedulingDiagnostics.createBuilder() + .setExecutionMode("adaptive") + .setMaxConcurrentTasks(maxConcurrentTasks) + .setCurrentConcurrentTasks(currentSettings.concurrentTasks()) + .setActiveProcessorInvocations(activeProcessorInvocations.get()) + .setCurrentRunDurationMillis(TimeUnit.NANOSECONDS.toMillis(currentSettings.runDurationNanos())) + .setConcurrencyEvaluationState(evaluationState) + .setConcurrencyUpdateReason(concurrencyUpdateStatus.reason().name()) + .setFlowFilesPerSecond(throughput) + .setMeasurementWindowMillis(TimeUnit.NANOSECONDS.toMillis(measurementWindowNanos)) + .setLastConcurrencyUpdateReason(lastChangeReason) + .setCollectingMeasurements(reason == ProcessorSchedulingDecisionReason.COLLECTING_COMPARISON_MEASUREMENTS) + .setFlowFileMeasurementsAvailable(measurementsSupported) + .setConcurrencyIncreaseExplanation(concurrencyUpdateStatus.explanation()) + .build(); + } + } + + private class SchedulingGeneration { private final CountDownLatch stopSignal = new CountDownLatch(1); private final Set threads = ConcurrentHashMap.newKeySet(); + private final Set waiters = ConcurrentHashMap.newKeySet(); + private final Set concurrentTaskSlotWaiters = ConcurrentHashMap.newKeySet(); + private final Set queues = ConcurrentHashMap.newKeySet(); + private final List queueRegistrations = new ArrayList<>(); private final AtomicBoolean interruptRequested = new AtomicBoolean(); + private final AtomicLong changeSequence = new AtomicLong(); + private final AtomicLong concurrentTaskSlotChangeCount = new AtomicLong(); + private volatile ProcessorAutoSchedulingState autoSchedulingState; + + void setAutoSchedulingState(final ProcessorAutoSchedulingState autoSchedulingState) { + this.autoSchedulingState = autoSchedulingState; + } + + ProcessorAutoSchedulingState getAutoSchedulingState() { + return autoSchedulingState; + } + + void addQueue(final FlowFileQueue queue) { + queues.add(queue); + } + + synchronized void addQueueRegistration(final QueueSchedulingRegistration registration) { + if (isStopped()) { + registration.close(); + } else { + queueRegistrations.add(registration); + } + } + + synchronized void clearQueueRegistrations() { + for (final QueueSchedulingRegistration registration : queueRegistrations) { + registration.close(); + } + + queueRegistrations.clear(); + queues.clear(); + } + + long getChangeSequence() { + return changeSequence.get(); + } + + int getWaiterCount() { + return waiters.size(); + } + + long getConcurrentTaskSlotChangeCount() { + return concurrentTaskSlotChangeCount.get(); + } + + int getConcurrentTaskSlotWaiterCount() { + return concurrentTaskSlotWaiters.size(); + } + + void signalChange() { + changeSequence.incrementAndGet(); + final ProcessorAutoSchedulingState state = autoSchedulingState; + final int wakeLimit = state == null ? waiters.size() : state.schedulingSettings.get().concurrentTasks(); + int awakened = 0; + for (final WaitingProcessorTask waiter : waiters) { + if (waiter.worker().retired.get()) { + waiters.remove(waiter); + continue; + } + + LockSupport.unpark(waiter.thread()); + if (++awakened >= wakeLimit) { + break; + } + } + } + + void awaitChange(final long expectedSequence, final long delayNanos, final ProcessorTaskWorker worker) { + final Thread currentThread = Thread.currentThread(); + final WaitingProcessorTask waiter = new WaitingProcessorTask(currentThread, worker); + waiters.add(waiter); + try { + if (!isStopped() && !worker.retired.get() && changeSequence.get() == expectedSequence) { + LockSupport.parkNanos(Math.max(1L, delayNanos)); + } + } finally { + waiters.remove(waiter); + } + } + + void signalConcurrentTaskSlotChange() { + concurrentTaskSlotChangeCount.incrementAndGet(); + // Retired waiters are woken as well so that they observe retirement and exit. + for (final WaitingProcessorTask waiter : concurrentTaskSlotWaiters) { + LockSupport.unpark(waiter.thread()); + } + } + + void waitForConcurrentTaskSlot(final long expectedChangeCount, final ProcessorTaskWorker worker) { + final Thread currentThread = Thread.currentThread(); + final WaitingProcessorTask waiter = new WaitingProcessorTask(currentThread, worker); + concurrentTaskSlotWaiters.add(waiter); + try { + while (!isStopped() && !worker.retired.get() && concurrentTaskSlotChangeCount.get() == expectedChangeCount) { + LockSupport.park(); + } + } finally { + concurrentTaskSlotWaiters.remove(waiter); + } + } void addThread(final Thread thread) { threads.add(thread); @@ -608,6 +1382,9 @@ void stop(final boolean interrupt) { } stopSignal.countDown(); + signalChange(); + signalConcurrentTaskSlotChange(); + clearQueueRegistrations(); if (interruptRequested.get()) { for (final Thread thread : threads) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingDiagnostics.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingDiagnostics.java new file mode 100644 index 000000000000..134b9da12737 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingDiagnostics.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +public final class AutoSchedulingDiagnostics { + private final String executionMode; + private final int maxConcurrentTasks; + private final int currentConcurrentTasks; + private final int activeProcessorInvocations; + private final long currentRunDurationMillis; + private final String concurrencyEvaluationState; + private final String concurrencyUpdateReason; + private final double flowFilesPerSecond; + private final long measurementWindowMillis; + private final String lastConcurrencyUpdateReason; + private final boolean collectingMeasurements; + private final boolean flowFileMeasurementsAvailable; + private final String concurrencyIncreaseExplanation; + + private AutoSchedulingDiagnostics(final Builder builder) { + executionMode = builder.executionMode; + maxConcurrentTasks = builder.maxConcurrentTasks; + currentConcurrentTasks = builder.currentConcurrentTasks; + activeProcessorInvocations = builder.activeProcessorInvocations; + currentRunDurationMillis = builder.currentRunDurationMillis; + concurrencyEvaluationState = builder.concurrencyEvaluationState; + concurrencyUpdateReason = builder.concurrencyUpdateReason; + flowFilesPerSecond = builder.flowFilesPerSecond; + measurementWindowMillis = builder.measurementWindowMillis; + lastConcurrencyUpdateReason = builder.lastConcurrencyUpdateReason; + collectingMeasurements = builder.collectingMeasurements; + flowFileMeasurementsAvailable = builder.flowFileMeasurementsAvailable; + concurrencyIncreaseExplanation = builder.concurrencyIncreaseExplanation; + } + + public static Builder createBuilder() { + return new Builder(); + } + + public String executionMode() { + return executionMode; + } + + public int maxConcurrentTasks() { + return maxConcurrentTasks; + } + + public int currentConcurrentTasks() { + return currentConcurrentTasks; + } + + public int activeProcessorInvocations() { + return activeProcessorInvocations; + } + + public long currentRunDurationMillis() { + return currentRunDurationMillis; + } + + public String concurrencyEvaluationState() { + return concurrencyEvaluationState; + } + + public String concurrencyUpdateReason() { + return concurrencyUpdateReason; + } + + public double flowFilesPerSecond() { + return flowFilesPerSecond; + } + + public long measurementWindowMillis() { + return measurementWindowMillis; + } + + public String lastConcurrencyUpdateReason() { + return lastConcurrencyUpdateReason; + } + + public boolean collectingMeasurements() { + return collectingMeasurements; + } + + public boolean flowFileMeasurementsAvailable() { + return flowFileMeasurementsAvailable; + } + + public String concurrencyIncreaseExplanation() { + return concurrencyIncreaseExplanation; + } + + public static final class Builder { + private String executionMode; + private int maxConcurrentTasks; + private int currentConcurrentTasks; + private int activeProcessorInvocations; + private long currentRunDurationMillis; + private String concurrencyEvaluationState; + private String concurrencyUpdateReason; + private double flowFilesPerSecond; + private long measurementWindowMillis; + private String lastConcurrencyUpdateReason; + private boolean collectingMeasurements; + private boolean flowFileMeasurementsAvailable; + private String concurrencyIncreaseExplanation; + + private Builder() { + } + + public Builder setExecutionMode(final String executionMode) { + this.executionMode = executionMode; + return this; + } + + public Builder setMaxConcurrentTasks(final int maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + return this; + } + + public Builder setCurrentConcurrentTasks(final int currentConcurrentTasks) { + this.currentConcurrentTasks = currentConcurrentTasks; + return this; + } + + public Builder setActiveProcessorInvocations(final int activeProcessorInvocations) { + this.activeProcessorInvocations = activeProcessorInvocations; + return this; + } + + public Builder setCurrentRunDurationMillis(final long currentRunDurationMillis) { + this.currentRunDurationMillis = currentRunDurationMillis; + return this; + } + + public Builder setConcurrencyEvaluationState(final String concurrencyEvaluationState) { + this.concurrencyEvaluationState = concurrencyEvaluationState; + return this; + } + + public Builder setConcurrencyUpdateReason(final String concurrencyUpdateReason) { + this.concurrencyUpdateReason = concurrencyUpdateReason; + return this; + } + + public Builder setFlowFilesPerSecond(final double flowFilesPerSecond) { + this.flowFilesPerSecond = flowFilesPerSecond; + return this; + } + + public Builder setMeasurementWindowMillis(final long measurementWindowMillis) { + this.measurementWindowMillis = measurementWindowMillis; + return this; + } + + public Builder setLastConcurrencyUpdateReason(final String lastConcurrencyUpdateReason) { + this.lastConcurrencyUpdateReason = lastConcurrencyUpdateReason; + return this; + } + + public Builder setCollectingMeasurements(final boolean collectingMeasurements) { + this.collectingMeasurements = collectingMeasurements; + return this; + } + + public Builder setFlowFileMeasurementsAvailable(final boolean flowFileMeasurementsAvailable) { + this.flowFileMeasurementsAvailable = flowFileMeasurementsAvailable; + return this; + } + + public Builder setConcurrencyIncreaseExplanation(final String concurrencyIncreaseExplanation) { + this.concurrencyIncreaseExplanation = concurrencyIncreaseExplanation; + return this; + } + + public AutoSchedulingDiagnostics build() { + return new AutoSchedulingDiagnostics(this); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingResetReason.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingResetReason.java new file mode 100644 index 000000000000..104d22cbde68 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingResetReason.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +public enum AutoSchedulingResetReason { + PROCESSOR_CONNECTIONS_CHANGED, + GLOBAL_CONCURRENT_TASK_LIMIT_CHANGED +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyEvaluationState.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyEvaluationState.java new file mode 100644 index 000000000000..52025434412d --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyEvaluationState.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +public enum ConcurrencyEvaluationState { + USING_CURRENT_CONCURRENCY, + MEASURING_CONCURRENCY_CHANGE +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyUpdateReason.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyUpdateReason.java new file mode 100644 index 000000000000..4e1e71f8a22f --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyUpdateReason.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +public enum ConcurrencyUpdateReason { + COLLECTING_PERFORMANCE_MEASUREMENTS, + REQUIRES_SINGLE_CONCURRENT_TASK, + CONCURRENT_TASK_LIMIT_REACHED, + CPU_LIMIT_REACHED, + CONFIGURED_MAX_CONCURRENT_TASKS_REACHED, + TESTING_HIGHER_CONCURRENCY, + TESTING_LOWER_CONCURRENCY, + PROCESSOR_INVOCATION_FAILED, + GARBAGE_COLLECTION_TIME_TOO_HIGH, + CPU_LOAD_TOO_HIGH, + GLOBAL_CAPACITY_FULL, + NO_WAITING_WORK, + PROCESSOR_NOT_READY, + DOWNSTREAM_BACK_PRESSURE, + PROCESSOR_YIELDING, + CONCURRENT_TASKS_UNDERUSED, + PREVIOUS_INCREASE_DID_NOT_IMPROVE_THROUGHPUT, + COOLDOWN +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyUpdateStatus.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyUpdateStatus.java new file mode 100644 index 000000000000..4c2e4357e8e0 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrencyUpdateStatus.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import java.util.Objects; + +public record ConcurrencyUpdateStatus(ConcurrencyUpdateReason reason, String explanation) { + + public ConcurrencyUpdateStatus { + Objects.requireNonNull(reason, "Concurrency update reason is required"); + Objects.requireNonNull(explanation, "Concurrency update explanation is required"); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingDecision.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingDecision.java new file mode 100644 index 000000000000..42342098eb9b --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingDecision.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.SchedulingSettings; + +import java.util.Objects; + +public record ProcessorSchedulingDecision(SchedulingSettings settings, ConcurrencyEvaluationState state, + ProcessorSchedulingDecisionReason reason, boolean applySettings) { + + public ProcessorSchedulingDecision { + Objects.requireNonNull(settings, "Scheduling settings are required"); + Objects.requireNonNull(state, "Concurrency evaluation state is required"); + Objects.requireNonNull(reason, "Decision reason is required"); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingDecisionReason.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingDecisionReason.java new file mode 100644 index 000000000000..e314522e1eb8 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingDecisionReason.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +public enum ProcessorSchedulingDecisionReason { + COLLECTING_COMPARISON_MEASUREMENTS, + TESTING_HIGHER_CONCURRENCY, + TESTING_LOWER_CONCURRENCY, + HIGHER_CONCURRENCY_INCREASED_THROUGHPUT, + HIGHER_CONCURRENCY_DID_NOT_INCREASE_THROUGHPUT, + LOWER_CONCURRENCY_MAINTAINED_THROUGHPUT, + LOWER_CONCURRENCY_REDUCED_THROUGHPUT_OR_INCREASED_INPUT_QUEUE, + COMPARISON_EXPIRED_WITHOUT_ENOUGH_MEASUREMENTS, + CPU_LOAD_TOO_HIGH, + GARBAGE_COLLECTION_TIME_TOO_HIGH, + PROCESSOR_INVOCATION_FAILED, + GLOBAL_CAPACITY_FULL, + NO_WAITING_WORK, + REDUCED_UNUSED_CONCURRENT_TASK, + WAITING_BEFORE_ANOTHER_INCREASE, + CONCURRENT_TASKS_EXCEEDED_CURRENT_LIMIT, + PERFORMANCE_MEASUREMENTS_RESET, + PROCESSOR_BLOCKED_BY_BACK_PRESSURE, + PROCESSOR_YIELDING +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurements.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurements.java new file mode 100644 index 000000000000..2436bdff1a4e --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurements.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.CommittedSchedulingWork; +import org.apache.nifi.controller.scheduling.SchedulingSettings; +import org.apache.nifi.controller.tasks.InvocationObserver; +import org.apache.nifi.controller.tasks.InvocationOutcome; +import org.apache.nifi.controller.tasks.InvocationResult; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.LongAdder; +import java.util.function.LongSupplier; + +public class ProcessorSchedulingMeasurements { + private final LongAdder committedFlowFiles = new LongAdder(); + private final LongAdder completedInvocations = new LongAdder(); + private final LongAdder failedInvocations = new LongAdder(); + private final LongAdder processorInvocationNanos = new LongAdder(); + private final LongSupplier nanoTimeSupplier; + private final Object activeInvocationLock = new Object(); + private int activeInvocations; + private long activeInvocationChangeNanos; + private long activeInvocationWindowStartNanos; + private long activeInvocationNanos; + private long previousCommittedFlowFiles; + private long previousCompletedInvocations; + private long previousFailedInvocations; + private long previousProcessorInvocationNanos; + private long backpressureNanos; + private long processorYieldNanos; + + public ProcessorSchedulingMeasurements() { + this(System::nanoTime); + } + + ProcessorSchedulingMeasurements(final LongSupplier nanoTimeSupplier) { + this.nanoTimeSupplier = nanoTimeSupplier; + activeInvocationChangeNanos = nanoTimeSupplier.getAsLong(); + activeInvocationWindowStartNanos = activeInvocationChangeNanos; + } + + public InvocationObserver createInvocationObserver() { + return new InvocationMeasurementObserver(); + } + + public void recordInvocationDuration(final long nanos) { + processorInvocationNanos.add(Math.max(0L, nanos)); + } + + public void recordInvocationStarted() { + synchronized (activeInvocationLock) { + accumulateActiveInvocationNanos(); + activeInvocations++; + } + } + + public void recordInvocationFinished() { + synchronized (activeInvocationLock) { + accumulateActiveInvocationNanos(); + activeInvocations--; + } + } + + public void recordReadiness(final InvocationOutcome readinessOutcome, final long measurementNanos) { + if (readinessOutcome == InvocationOutcome.BACKPRESSURED) { + backpressureNanos += measurementNanos; + } else if (readinessOutcome == InvocationOutcome.YIELDED) { + processorYieldNanos += measurementNanos; + } + } + + /** + * Captures the measurements recorded since the previous snapshot. Concurrent task utilization is the total time spent in + * invocations since the previous snapshot, including invocations that are still running, divided by the time available + * to the allowed concurrent tasks over that same period. Utilization periods begin and end at times read while holding the + * lock that also guards invocation start and finish, so each moment of invocation time belongs to exactly one snapshot even + * when the caller's timestamp was read before an invocation started. + */ + public ProcessorSchedulingSnapshot captureSnapshot(final long timestampNanos, final long measurementWindowNanos, final SchedulingSettings settings, + final boolean processorReady, final boolean inputQueueHasFlowFiles, final double inputQueueGrowth, + final boolean sourceProcessorRecentlyReportedActivity, final boolean primaryNodeChanged) { + final long committedFlowFileCount = committedFlowFiles.sum(); + final long completedInvocationCount = completedInvocations.sum(); + final long failedInvocationCount = failedInvocations.sum(); + final long processorInvocationDurationNanos = processorInvocationNanos.sum(); + final long windowActiveInvocationNanos; + final long utilizationWindowNanos; + synchronized (activeInvocationLock) { + accumulateActiveInvocationNanos(); + windowActiveInvocationNanos = activeInvocationNanos; + utilizationWindowNanos = activeInvocationChangeNanos - activeInvocationWindowStartNanos; + activeInvocationNanos = 0L; + activeInvocationWindowStartNanos = activeInvocationChangeNanos; + } + + final double concurrentTaskUtilization = Math.min(1D, + windowActiveInvocationNanos / ((double) Math.max(1L, utilizationWindowNanos) * settings.concurrentTasks())); + final ProcessorSchedulingSnapshot snapshot = ProcessorSchedulingSnapshot.createBuilder() + .setTimestampNanos(timestampNanos) + .setMeasurementWindowNanos(measurementWindowNanos) + .setAppliedSettings(settings) + .setCommittedFlowFiles(committedFlowFileCount - previousCommittedFlowFiles) + .setCompletedInvocations(completedInvocationCount - previousCompletedInvocations) + .setFailedInvocations(failedInvocationCount - previousFailedInvocations) + .setProcessorInvocationNanos(processorInvocationDurationNanos - previousProcessorInvocationNanos) + .setBackpressureNanos(backpressureNanos) + .setProcessorYieldNanos(processorYieldNanos) + .setConcurrentTaskUtilization(concurrentTaskUtilization) + .setProcessorReady(processorReady) + .setInputQueueHasFlowFiles(inputQueueHasFlowFiles) + .setInputQueueGrowth(inputQueueGrowth) + .setSourceProcessorRecentlyReportedActivity(sourceProcessorRecentlyReportedActivity) + .setPrimaryNodeChanged(primaryNodeChanged) + .build(); + previousCommittedFlowFiles = committedFlowFileCount; + previousCompletedInvocations = completedInvocationCount; + previousFailedInvocations = failedInvocationCount; + previousProcessorInvocationNanos = processorInvocationDurationNanos; + backpressureNanos = 0L; + processorYieldNanos = 0L; + return snapshot; + } + + /** + * Adds the invocation time since the last change. Must be called while holding the active invocation lock; reading the + * clock under the lock keeps the change timestamp from moving backward. + */ + private void accumulateActiveInvocationNanos() { + final long nowNanos = nanoTimeSupplier.getAsLong(); + activeInvocationNanos += activeInvocations * (nowNanos - activeInvocationChangeNanos); + activeInvocationChangeNanos = nowNanos; + } + + private void recordCompletedInvocation(final InvocationOutcome outcome) { + if (outcome == InvocationOutcome.INVOKED_WITH_ACTIVITY || outcome == InvocationOutcome.INVOKED_WITHOUT_ACTIVITY + || outcome == InvocationOutcome.FAILED) { + completedInvocations.increment(); + } + } + + private long recordCommittedFlowFiles(final CommittedSchedulingWork committedWork) { + // Count committed input FlowFiles when present, otherwise count committed produced FlowFiles for source processors. + final long count = committedWork.inputFlowFiles() > 0L ? committedWork.inputFlowFiles() : committedWork.producedFlowFiles(); + committedFlowFiles.add(count); + return count; + } + + /** + * Observes a single invocation. The invocation is counted as failed at most once, whether the failure is reported by a + * failed session commit, by the invocation result, or by both, so that failed invocations and completed invocations count + * the same thing. + */ + private class InvocationMeasurementObserver implements InvocationObserver { + private final AtomicBoolean failureRecorded = new AtomicBoolean(); + private volatile boolean activityObserved; + private volatile boolean triggerActivityObserved; + + @Override + public boolean isActivityObserved() { + return activityObserved; + } + + @Override + public void resetTriggerActivity() { + triggerActivityObserved = false; + } + + @Override + public boolean isTriggerActivityObserved() { + return triggerActivityObserved; + } + + @Override + public void onInvocationCompleted(final InvocationResult result) { + recordCompletedInvocation(result.getOutcome()); + if (result.getOutcome() == InvocationOutcome.FAILED) { + recordFailure(); + } + } + + @Override + public void onActivity() { + activityObserved = true; + triggerActivityObserved = true; + } + + @Override + public void onCommit(final CommittedSchedulingWork committedWork) { + final long count = recordCommittedFlowFiles(committedWork); + if (count > 0L) { + onActivity(); + } + } + + @Override + public void onCommitFailure() { + recordFailure(); + } + + private void recordFailure() { + if (failureRecorded.compareAndSet(false, true)) { + failedInvocations.increment(); + } + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingSnapshot.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingSnapshot.java new file mode 100644 index 000000000000..c91c711b8c4c --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingSnapshot.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.SchedulingSettings; + +public final class ProcessorSchedulingSnapshot { + private final long timestampNanos; + private final long measurementWindowNanos; + private final SchedulingSettings appliedSettings; + private final long committedFlowFiles; + private final long completedInvocations; + private final long failedInvocations; + private final long processorInvocationNanos; + private final long backpressureNanos; + private final long processorYieldNanos; + private final double concurrentTaskUtilization; + private final boolean processorReady; + private final boolean inputQueueHasFlowFiles; + private final double inputQueueGrowth; + private final boolean sourceProcessorRecentlyReportedActivity; + private final boolean primaryNodeChanged; + + private ProcessorSchedulingSnapshot(final Builder builder) { + timestampNanos = builder.timestampNanos; + measurementWindowNanos = builder.measurementWindowNanos; + appliedSettings = builder.appliedSettings; + committedFlowFiles = builder.committedFlowFiles; + completedInvocations = builder.completedInvocations; + failedInvocations = builder.failedInvocations; + processorInvocationNanos = builder.processorInvocationNanos; + backpressureNanos = builder.backpressureNanos; + processorYieldNanos = builder.processorYieldNanos; + concurrentTaskUtilization = builder.concurrentTaskUtilization; + processorReady = builder.processorReady; + inputQueueHasFlowFiles = builder.inputQueueHasFlowFiles; + inputQueueGrowth = builder.inputQueueGrowth; + sourceProcessorRecentlyReportedActivity = builder.sourceProcessorRecentlyReportedActivity; + primaryNodeChanged = builder.primaryNodeChanged; + } + + public static Builder createBuilder() { + return new Builder(); + } + + public long timestampNanos() { + return timestampNanos; + } + + public long measurementWindowNanos() { + return measurementWindowNanos; + } + + public SchedulingSettings appliedSettings() { + return appliedSettings; + } + + public long committedFlowFiles() { + return committedFlowFiles; + } + + public long completedInvocations() { + return completedInvocations; + } + + public long failedInvocations() { + return failedInvocations; + } + + public long processorInvocationNanos() { + return processorInvocationNanos; + } + + public long backpressureNanos() { + return backpressureNanos; + } + + public long processorYieldNanos() { + return processorYieldNanos; + } + + public double concurrentTaskUtilization() { + return concurrentTaskUtilization; + } + + public boolean processorReady() { + return processorReady; + } + + public boolean inputQueueHasFlowFiles() { + return inputQueueHasFlowFiles; + } + + public double inputQueueGrowth() { + return inputQueueGrowth; + } + + public boolean sourceProcessorRecentlyReportedActivity() { + return sourceProcessorRecentlyReportedActivity; + } + + public boolean primaryNodeChanged() { + return primaryNodeChanged; + } + + public static final class Builder { + private long timestampNanos; + private long measurementWindowNanos; + private SchedulingSettings appliedSettings; + private long committedFlowFiles; + private long completedInvocations; + private long failedInvocations; + private long processorInvocationNanos; + private long backpressureNanos; + private long processorYieldNanos; + private double concurrentTaskUtilization; + private boolean processorReady; + private boolean inputQueueHasFlowFiles; + private double inputQueueGrowth; + private boolean sourceProcessorRecentlyReportedActivity; + private boolean primaryNodeChanged; + + private Builder() { + } + + public Builder setTimestampNanos(final long timestampNanos) { + this.timestampNanos = timestampNanos; + return this; + } + + public Builder setMeasurementWindowNanos(final long measurementWindowNanos) { + this.measurementWindowNanos = measurementWindowNanos; + return this; + } + + public Builder setAppliedSettings(final SchedulingSettings appliedSettings) { + this.appliedSettings = appliedSettings; + return this; + } + + public Builder setCommittedFlowFiles(final long committedFlowFiles) { + this.committedFlowFiles = committedFlowFiles; + return this; + } + + public Builder setCompletedInvocations(final long completedInvocations) { + this.completedInvocations = completedInvocations; + return this; + } + + public Builder setFailedInvocations(final long failedInvocations) { + this.failedInvocations = failedInvocations; + return this; + } + + public Builder setProcessorInvocationNanos(final long processorInvocationNanos) { + this.processorInvocationNanos = processorInvocationNanos; + return this; + } + + public Builder setBackpressureNanos(final long backpressureNanos) { + this.backpressureNanos = backpressureNanos; + return this; + } + + public Builder setProcessorYieldNanos(final long processorYieldNanos) { + this.processorYieldNanos = processorYieldNanos; + return this; + } + + public Builder setConcurrentTaskUtilization(final double concurrentTaskUtilization) { + this.concurrentTaskUtilization = concurrentTaskUtilization; + return this; + } + + public Builder setProcessorReady(final boolean processorReady) { + this.processorReady = processorReady; + return this; + } + + public Builder setInputQueueHasFlowFiles(final boolean inputQueueHasFlowFiles) { + this.inputQueueHasFlowFiles = inputQueueHasFlowFiles; + return this; + } + + public Builder setInputQueueGrowth(final double inputQueueGrowth) { + this.inputQueueGrowth = inputQueueGrowth; + return this; + } + + public Builder setSourceProcessorRecentlyReportedActivity(final boolean sourceProcessorRecentlyReportedActivity) { + this.sourceProcessorRecentlyReportedActivity = sourceProcessorRecentlyReportedActivity; + return this; + } + + public Builder setPrimaryNodeChanged(final boolean primaryNodeChanged) { + this.primaryNodeChanged = primaryNodeChanged; + return this; + } + + public ProcessorSchedulingSnapshot build() { + return new ProcessorSchedulingSnapshot(this); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingController.java new file mode 100644 index 000000000000..34069b4db581 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingController.java @@ -0,0 +1,479 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.SchedulingSettings; + +import java.util.concurrent.TimeUnit; + +/** + * Chooses how many concurrent processor calls should be allowed. + * + *

The controller starts with one concurrent call. It watches completed work, available processor work, + * how much of the time the allowed concurrent tasks are in use, back pressure, processor yields, the rate of failed calls, CPU load, + * garbage collection, and the global concurrent task limit. It changes the setting only after collecting + * enough measurements to make a useful comparison. Each measurement period lasts long enough for every concurrent call + * to complete several invocations, from 6 seconds for fast processors up to 30 seconds for slow processors.

+ * + *

When there is sustained work and the existing calls are busy, the controller tests a higher number of + * concurrent calls. Each test adds one call, and the higher number is kept only when it produces a clear increase + * in completed work. No increase exceeds the processor maximum or the global concurrent task limit. An unsuccessful + * increase waits before it is tried again.

+ * + *

The controller also looks for a lower number of calls. If the processor is mostly idle or blocked for several + * seconds, it removes one unused call immediately. At regular intervals it tests one fewer call even when the + * existing calls are busy. That lower setting is kept only when completed work stays close to the reference rate and + * the input queue does not grow. The reference rate is the rate measured at the highest setting that proved useful, so + * a series of lower-setting tests cannot each give up a little more throughput than the one before. An unsuccessful + * lower-setting test leaves the current setting in place and replaces the reference rate with the current rate, so that + * a reference rate from an earlier, heavier workload does not block lower-setting tests indefinitely.

+ * + *

The run duration is fixed when this controller is created. Batching support uses a 25 millisecond duration; + * processors without batching use no added duration.

+ */ +public class StandardAutoSchedulingController { + private static final long MIN_COMPARISON_NANOS = TimeUnit.SECONDS.toNanos(6); + private static final long MAX_COMPARISON_NANOS = TimeUnit.SECONDS.toNanos(30); + private static final int COMPARISON_INVOCATIONS_PER_TASK = 10; + private static final long INCREASE_COOLDOWN_NANOS = TimeUnit.SECONDS.toNanos(10); + private static final long REDUCTION_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); + private static final long UNUSED_TASK_REDUCTION_NANOS = TimeUnit.SECONDS.toNanos(3); + private static final double THROUGHPUT_TOLERANCE = 0.05D; + private static final double MAX_FAILED_INVOCATION_FRACTION = 0.10D; + + private final int maxConcurrentTasks; + private final long runDurationNanos; + private final boolean processorTriggeredSerially; + private final boolean maxConcurrentTasksBasedOnAvailableProcessors; + private final int availableProcessorCount; + private ConcurrencyComparison concurrencyComparison; + private double baselineMeasuredFlowFileCount; + private double baselineMeasurementNanos; + private double referenceFlowFilesPerSecond = -1D; + private long averageInvocationNanos; + private long nextIncreaseNanos; + private long nextReductionNanos; + private long concurrentTasksUnderusedNanos; + private boolean previousIncreaseDidNotImproveThroughput; + private volatile ConcurrencyUpdateStatus concurrencyUpdateStatus = new ConcurrencyUpdateStatus( + ConcurrencyUpdateReason.COLLECTING_PERFORMANCE_MEASUREMENTS, + "Gathering performance measurements before trying to increase concurrent tasks."); + + public StandardAutoSchedulingController(final int maxConcurrentTasks, final boolean batchingSupported, + final boolean processorTriggeredSerially, + final boolean maxConcurrentTasksBasedOnAvailableProcessors, + final int availableProcessorCount) { + this.maxConcurrentTasks = processorTriggeredSerially ? 1 : maxConcurrentTasks; + runDurationNanos = batchingSupported ? TimeUnit.MILLISECONDS.toNanos(25) : 0L; + this.processorTriggeredSerially = processorTriggeredSerially; + this.maxConcurrentTasksBasedOnAvailableProcessors = maxConcurrentTasksBasedOnAvailableProcessors; + this.availableProcessorCount = availableProcessorCount; + } + + /** + * Reviews the latest processor and system measurements and determines whether the current settings should change. + * + * @param processorMeasurements measurements for this processor + * @param systemMeasurements measurements for the system and its global concurrent task limit + * @return the current settings or settings to apply + */ + public ProcessorSchedulingDecision evaluate(final ProcessorSchedulingSnapshot processorMeasurements, + final SystemSchedulingSnapshot systemMeasurements) { + final SchedulingSettings currentSettings = processorMeasurements.appliedSettings(); + final long timestampNanos = processorMeasurements.timestampNanos(); + final int maxAllowedConcurrentTasks = Math.max(1, Math.min(maxConcurrentTasks, systemMeasurements.maxGlobalConcurrentTasks())); + if (currentSettings.concurrentTasks() > maxAllowedConcurrentTasks || currentSettings.runDurationNanos() != runDurationNanos) { + reset(); + return createDecision(Math.min(currentSettings.concurrentTasks(), maxAllowedConcurrentTasks), + ProcessorSchedulingDecisionReason.CONCURRENT_TASKS_EXCEEDED_CURRENT_LIMIT, true, processorMeasurements, systemMeasurements); + } + + if (processorMeasurements.primaryNodeChanged()) { + reset(); + } + + if (processorMeasurements.completedInvocations() > 0L && processorMeasurements.processorInvocationNanos() > 0L) { + averageInvocationNanos = processorMeasurements.processorInvocationNanos() / processorMeasurements.completedInvocations(); + } + + if (concurrencyComparison != null) { + return evaluateConcurrencyComparison(processorMeasurements, systemMeasurements); + } + + final boolean backpressureApplied = isBackpressureApplied(processorMeasurements); + final boolean processorYielding = isProcessorYielding(processorMeasurements); + + baselineMeasuredFlowFileCount += processorMeasurements.committedFlowFiles(); + baselineMeasurementNanos += processorMeasurements.measurementWindowNanos(); + final long baselineWindowNanos = getComparisonWindowNanos(); + if (baselineMeasurementNanos > baselineWindowNanos * 2) { + baselineMeasuredFlowFileCount *= 0.5D; + baselineMeasurementNanos *= 0.5D; + } + + if (nextReductionNanos == 0L) { + nextReductionNanos = timestampNanos + REDUCTION_INTERVAL_NANOS; + } + + final boolean concurrentTasksUnderused = processorMeasurements.concurrentTaskUtilization() < 0.5D; + concurrentTasksUnderusedNanos = concurrentTasksUnderused || backpressureApplied || processorYielding + ? concurrentTasksUnderusedNanos + processorMeasurements.measurementWindowNanos() : 0L; + if (currentSettings.concurrentTasks() > 1 && concurrentTasksUnderusedNanos >= UNUSED_TASK_REDUCTION_NANOS) { + concurrentTasksUnderusedNanos = 0L; + referenceFlowFilesPerSecond = -1D; + return createDecision(currentSettings.concurrentTasks() - 1, ProcessorSchedulingDecisionReason.REDUCED_UNUSED_CONCURRENT_TASK, + true, processorMeasurements, systemMeasurements); + } + + if (currentSettings.concurrentTasks() > 1 && timestampNanos >= nextReductionNanos) { + return startConcurrencyComparison(processorMeasurements, systemMeasurements, currentSettings.concurrentTasks() - 1); + } + + final ConcurrencyUpdateReason increaseBlocker = findIncreaseBlocker(currentSettings.concurrentTasks(), processorMeasurements, systemMeasurements); + if (increaseBlocker == null) { + return startConcurrencyComparison(processorMeasurements, systemMeasurements, currentSettings.concurrentTasks() + 1); + } + + final ProcessorSchedulingDecisionReason reason = switch (increaseBlocker) { + case PROCESSOR_INVOCATION_FAILED -> ProcessorSchedulingDecisionReason.PROCESSOR_INVOCATION_FAILED; + case CPU_LOAD_TOO_HIGH -> ProcessorSchedulingDecisionReason.CPU_LOAD_TOO_HIGH; + case GARBAGE_COLLECTION_TIME_TOO_HIGH -> ProcessorSchedulingDecisionReason.GARBAGE_COLLECTION_TIME_TOO_HIGH; + case GLOBAL_CAPACITY_FULL -> ProcessorSchedulingDecisionReason.GLOBAL_CAPACITY_FULL; + case NO_WAITING_WORK -> ProcessorSchedulingDecisionReason.NO_WAITING_WORK; + default -> ProcessorSchedulingDecisionReason.WAITING_BEFORE_ANOTHER_INCREASE; + }; + return createDecision(currentSettings.concurrentTasks(), reason, false, processorMeasurements, systemMeasurements); + } + + public void reset() { + concurrencyComparison = null; + baselineMeasuredFlowFileCount = 0D; + baselineMeasurementNanos = 0D; + referenceFlowFilesPerSecond = -1D; + averageInvocationNanos = 0L; + nextIncreaseNanos = 0L; + nextReductionNanos = 0L; + concurrentTasksUnderusedNanos = 0L; + previousIncreaseDidNotImproveThroughput = false; + } + + public ConcurrencyUpdateStatus getConcurrencyUpdateStatus() { + return concurrencyUpdateStatus; + } + + /** + * Chooses a measurement window long enough for each concurrent task to complete about + * {@value #COMPARISON_INVOCATIONS_PER_TASK} invocations, so that partial invocations at the edges of the window + * have little effect on the measured rate. The window is limited to the minimum and maximum comparison durations. + */ + private long getComparisonWindowNanos() { + return Math.max(MIN_COMPARISON_NANOS, Math.min(MAX_COMPARISON_NANOS, averageInvocationNanos * COMPARISON_INVOCATIONS_PER_TASK)); + } + + private ProcessorSchedulingDecision startConcurrencyComparison(final ProcessorSchedulingSnapshot processorMeasurements, + final SystemSchedulingSnapshot systemMeasurements, + final int comparisonConcurrentTasks) { + final long minMeasurementNanos = getComparisonWindowNanos(); + final double baselineFlowFilesPerSecond = baselineMeasuredFlowFileCount * TimeUnit.SECONDS.toNanos(1) + / Math.max(1D, baselineMeasurementNanos); + final int originalConcurrentTasks = processorMeasurements.appliedSettings().concurrentTasks(); + final boolean increasingConcurrentTasks = comparisonConcurrentTasks > originalConcurrentTasks; + final double originalFlowFilesPerSecond = increasingConcurrentTasks ? baselineFlowFilesPerSecond + : Math.max(baselineFlowFilesPerSecond, referenceFlowFilesPerSecond); + concurrencyComparison = new ConcurrencyComparison(originalConcurrentTasks, comparisonConcurrentTasks, originalFlowFilesPerSecond, + baselineFlowFilesPerSecond, processorMeasurements.timestampNanos(), minMeasurementNanos); + final ProcessorSchedulingDecisionReason reason = concurrencyComparison.isIncreasingConcurrentTasks() + ? ProcessorSchedulingDecisionReason.TESTING_HIGHER_CONCURRENCY : ProcessorSchedulingDecisionReason.TESTING_LOWER_CONCURRENCY; + return createDecision(comparisonConcurrentTasks, reason, true, processorMeasurements, systemMeasurements); + } + + private ProcessorSchedulingDecision evaluateConcurrencyComparison(final ProcessorSchedulingSnapshot processorMeasurements, + final SystemSchedulingSnapshot systemMeasurements) { + concurrencyComparison.addMeasurements(processorMeasurements); + if (hasHighFailureRate(processorMeasurements)) { + return finishConcurrencyComparison(false, ProcessorSchedulingDecisionReason.PROCESSOR_INVOCATION_FAILED, + processorMeasurements, systemMeasurements); + } + if (isBackpressureApplied(processorMeasurements)) { + return finishConcurrencyComparison(false, ProcessorSchedulingDecisionReason.PROCESSOR_BLOCKED_BY_BACK_PRESSURE, + processorMeasurements, systemMeasurements); + } + if (isProcessorYielding(processorMeasurements)) { + return finishConcurrencyComparison(false, ProcessorSchedulingDecisionReason.PROCESSOR_YIELDING, + processorMeasurements, systemMeasurements); + } + if (!concurrencyComparison.hasMinimumMeasurements()) { + if (concurrencyComparison.hasExpired(processorMeasurements.timestampNanos())) { + return finishConcurrencyComparison(false, ProcessorSchedulingDecisionReason.COMPARISON_EXPIRED_WITHOUT_ENOUGH_MEASUREMENTS, + processorMeasurements, systemMeasurements); + } + + return createDecision(concurrencyComparison.getComparisonConcurrentTasks(), + ProcessorSchedulingDecisionReason.COLLECTING_COMPARISON_MEASUREMENTS, false, processorMeasurements, systemMeasurements); + } + + final double tolerance = Math.min(THROUGHPUT_TOLERANCE, 0.5D / concurrencyComparison.getOriginalConcurrentTasks()); + final boolean accepted = concurrencyComparison.isIncreasingConcurrentTasks() + ? concurrencyComparison.hasIncreasedThroughput(tolerance) + : concurrencyComparison.hasMaintainedThroughput(tolerance) && !concurrencyComparison.hasInputQueueGrown(); + final ProcessorSchedulingDecisionReason reason; + if (concurrencyComparison.isIncreasingConcurrentTasks()) { + reason = accepted ? ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_INCREASED_THROUGHPUT + : ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_DID_NOT_INCREASE_THROUGHPUT; + } else { + reason = accepted ? ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_MAINTAINED_THROUGHPUT + : ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_REDUCED_THROUGHPUT_OR_INCREASED_INPUT_QUEUE; + } + + return finishConcurrencyComparison(accepted, reason, processorMeasurements, systemMeasurements); + } + + private ProcessorSchedulingDecision finishConcurrencyComparison(final boolean accepted, final ProcessorSchedulingDecisionReason reason, + final ProcessorSchedulingSnapshot processorMeasurements, + final SystemSchedulingSnapshot systemMeasurements) { + final boolean increasingConcurrentTasks = concurrencyComparison.isIncreasingConcurrentTasks(); + final int selectedConcurrentTasks = accepted ? concurrencyComparison.getComparisonConcurrentTasks() + : concurrencyComparison.getOriginalConcurrentTasks(); + if (accepted) { + baselineMeasuredFlowFileCount = concurrencyComparison.getMeasuredFlowFileCount(); + baselineMeasurementNanos = concurrencyComparison.getMeasurementWindowNanos(); + } + + if (increasingConcurrentTasks && accepted) { + referenceFlowFilesPerSecond = concurrencyComparison.getMeasuredFlowFilesPerSecond(); + } else if (!increasingConcurrentTasks) { + referenceFlowFilesPerSecond = accepted ? concurrencyComparison.getOriginalFlowFilesPerSecond() + : concurrencyComparison.getBaselineFlowFilesPerSecond(); + } + + final long timestampNanos = processorMeasurements.timestampNanos(); + nextIncreaseNanos = accepted && increasingConcurrentTasks ? timestampNanos : timestampNanos + INCREASE_COOLDOWN_NANOS; + previousIncreaseDidNotImproveThroughput = !accepted && increasingConcurrentTasks; + if (accepted || !increasingConcurrentTasks) { + nextReductionNanos = timestampNanos + + (accepted && !increasingConcurrentTasks ? UNUSED_TASK_REDUCTION_NANOS : REDUCTION_INTERVAL_NANOS); + } + + concurrencyComparison = null; + concurrentTasksUnderusedNanos = 0L; + return createDecision(selectedConcurrentTasks, reason, + selectedConcurrentTasks != processorMeasurements.appliedSettings().concurrentTasks(), processorMeasurements, systemMeasurements); + } + + private ProcessorSchedulingDecision createDecision(final int concurrentTasks, final ProcessorSchedulingDecisionReason reason, + final boolean applySettings, final ProcessorSchedulingSnapshot processorMeasurements, + final SystemSchedulingSnapshot systemMeasurements) { + final ConcurrencyUpdateReason increaseBlocker = findIncreaseBlocker(concurrentTasks, processorMeasurements, systemMeasurements); + concurrencyUpdateStatus = describeConcurrencyUpdate(increaseBlocker == null ? ConcurrencyUpdateReason.COLLECTING_PERFORMANCE_MEASUREMENTS : increaseBlocker, + concurrentTasks, processorMeasurements.timestampNanos(), systemMeasurements); + final ConcurrencyEvaluationState state = concurrencyComparison == null + ? ConcurrencyEvaluationState.USING_CURRENT_CONCURRENCY : ConcurrencyEvaluationState.MEASURING_CONCURRENCY_CHANGE; + return new ProcessorSchedulingDecision(new SchedulingSettings(concurrentTasks, runDurationNanos), state, reason, applySettings); + } + + /** + * Finds the first reason that prevents testing a higher number of concurrent tasks. The same reason determines both + * whether an increase is tested and the explanation reported to users. + * + * @return the reason an increase cannot be tested, or {@code null} if an increase can be tested now + */ + private ConcurrencyUpdateReason findIncreaseBlocker(final int concurrentTasks, final ProcessorSchedulingSnapshot processorMeasurements, + final SystemSchedulingSnapshot systemMeasurements) { + if (processorTriggeredSerially) { + return ConcurrencyUpdateReason.REQUIRES_SINGLE_CONCURRENT_TASK; + } + + if (concurrentTasks >= Math.min(maxConcurrentTasks, systemMeasurements.maxGlobalConcurrentTasks())) { + if (systemMeasurements.maxGlobalConcurrentTasks() < maxConcurrentTasks) { + return ConcurrencyUpdateReason.CONCURRENT_TASK_LIMIT_REACHED; + } + + return maxConcurrentTasksBasedOnAvailableProcessors ? ConcurrencyUpdateReason.CPU_LIMIT_REACHED + : ConcurrencyUpdateReason.CONFIGURED_MAX_CONCURRENT_TASKS_REACHED; + } + + if (concurrencyComparison != null) { + return concurrencyComparison.isIncreasingConcurrentTasks() ? ConcurrencyUpdateReason.TESTING_HIGHER_CONCURRENCY + : ConcurrencyUpdateReason.TESTING_LOWER_CONCURRENCY; + } + + if (hasHighFailureRate(processorMeasurements)) { + return ConcurrencyUpdateReason.PROCESSOR_INVOCATION_FAILED; + } + if (systemMeasurements.cpuLoadAboveLimit()) { + return ConcurrencyUpdateReason.CPU_LOAD_TOO_HIGH; + } + if (systemMeasurements.garbageCollectionTimeAboveLimit()) { + return ConcurrencyUpdateReason.GARBAGE_COLLECTION_TIME_TOO_HIGH; + } + if (systemMeasurements.fractionOfSamplesWithAllGlobalTaskSlotsUsed() >= 0.90D && systemMeasurements.fractionOfGlobalTaskTimeSpentWaiting() > 0.10D) { + return ConcurrencyUpdateReason.GLOBAL_CAPACITY_FULL; + } + if (!processorMeasurements.inputQueueHasFlowFiles() && !processorMeasurements.sourceProcessorRecentlyReportedActivity()) { + return ConcurrencyUpdateReason.NO_WAITING_WORK; + } + if (!processorMeasurements.processorReady()) { + return ConcurrencyUpdateReason.PROCESSOR_NOT_READY; + } + if (isBackpressureApplied(processorMeasurements)) { + return ConcurrencyUpdateReason.DOWNSTREAM_BACK_PRESSURE; + } + if (isProcessorYielding(processorMeasurements)) { + return ConcurrencyUpdateReason.PROCESSOR_YIELDING; + } + if (processorMeasurements.concurrentTaskUtilization() < 0.75D) { + return ConcurrencyUpdateReason.CONCURRENT_TASKS_UNDERUSED; + } + if (processorMeasurements.timestampNanos() < nextIncreaseNanos) { + return previousIncreaseDidNotImproveThroughput ? ConcurrencyUpdateReason.PREVIOUS_INCREASE_DID_NOT_IMPROVE_THROUGHPUT + : ConcurrencyUpdateReason.COOLDOWN; + } + if (concurrentTasks > 1 && baselineMeasurementNanos < getComparisonWindowNanos()) { + return ConcurrencyUpdateReason.COLLECTING_PERFORMANCE_MEASUREMENTS; + } + + return null; + } + + private ConcurrencyUpdateStatus describeConcurrencyUpdate(final ConcurrencyUpdateReason reason, final int concurrentTasks, final long timestampNanos, + final SystemSchedulingSnapshot systemMeasurements) { + final long cooldownSeconds = Math.max(1L, TimeUnit.NANOSECONDS.toSeconds(nextIncreaseNanos - timestampNanos + TimeUnit.SECONDS.toNanos(1L) - 1L)); + final String cooldownDescription = cooldownSeconds + (cooldownSeconds == 1L ? " second." : " seconds."); + final String explanation = switch (reason) { + case REQUIRES_SINGLE_CONCURRENT_TASK -> "Will not increase concurrent tasks because this Processor requires serial execution."; + case CONCURRENT_TASK_LIMIT_REACHED -> "Will not increase concurrent tasks because the instance has reached its concurrent task limit of " + + systemMeasurements.maxGlobalConcurrentTasks() + (systemMeasurements.maxGlobalConcurrentTasks() == 1 ? " concurrent task." : " concurrent tasks."); + case CPU_LIMIT_REACHED -> "Will not increase concurrent tasks because the Processor has reached the maximum of " + maxConcurrentTasks + + " concurrent tasks based on " + availableProcessorCount + (availableProcessorCount == 1 ? " available CPU core." : " available CPU cores."); + case CONFIGURED_MAX_CONCURRENT_TASKS_REACHED -> "Will not increase concurrent tasks because the Processor has reached the configured maximum of " + + maxConcurrentTasks + " concurrent tasks."; + case TESTING_HIGHER_CONCURRENCY -> "Testing whether " + concurrentTasks + " concurrent tasks improve performance before increasing further."; + case TESTING_LOWER_CONCURRENCY -> "Will not increase concurrent tasks while testing whether fewer concurrent tasks can maintain performance."; + case PROCESSOR_INVOCATION_FAILED -> "Will not increase concurrent tasks because too many recent Processor invocations failed."; + case CPU_LOAD_TOO_HIGH -> "Will not increase concurrent tasks because CPU load is too high."; + case GARBAGE_COLLECTION_TIME_TOO_HIGH -> "Will not increase concurrent tasks because garbage collection is using too much processing time."; + case GLOBAL_CAPACITY_FULL -> "Will not increase concurrent tasks because the instance's global scheduling capacity is fully used."; + case NO_WAITING_WORK -> "Will not increase concurrent tasks because there is not enough sustained work."; + case PROCESSOR_NOT_READY -> "Will not increase concurrent tasks because the Processor is not currently ready to run."; + case DOWNSTREAM_BACK_PRESSURE -> "Will not increase concurrent tasks because downstream back pressure is limiting the Processor."; + case PROCESSOR_YIELDING -> "Will not increase concurrent tasks because the Processor is yielding."; + case CONCURRENT_TASKS_UNDERUSED -> "Will not increase concurrent tasks because the current concurrent tasks are not busy enough."; + case PREVIOUS_INCREASE_DID_NOT_IMPROVE_THROUGHPUT -> "Will not increase concurrent tasks because the previous increase did not improve performance; " + + "will try again in " + cooldownDescription; + case COOLDOWN -> "Will not increase concurrent tasks during the cooldown; will try an increase in " + cooldownDescription; + case COLLECTING_PERFORMANCE_MEASUREMENTS -> "Gathering performance measurements before trying to increase concurrent tasks."; + }; + + return new ConcurrencyUpdateStatus(reason, explanation); + } + + private static boolean isBackpressureApplied(final ProcessorSchedulingSnapshot processorMeasurements) { + return processorMeasurements.backpressureNanos() >= processorMeasurements.measurementWindowNanos() / 2; + } + + private static boolean isProcessorYielding(final ProcessorSchedulingSnapshot processorMeasurements) { + return processorMeasurements.processorYieldNanos() >= processorMeasurements.measurementWindowNanos() / 2; + } + + private static boolean hasHighFailureRate(final ProcessorSchedulingSnapshot processorMeasurements) { + final long failedInvocations = processorMeasurements.failedInvocations(); + return failedInvocations > 0L && failedInvocations >= processorMeasurements.completedInvocations() * MAX_FAILED_INVOCATION_FRACTION; + } + + private static class ConcurrencyComparison { + private final int originalConcurrentTasks; + private final int comparisonConcurrentTasks; + private final double originalFlowFilesPerSecond; + private final double baselineFlowFilesPerSecond; + private final long comparisonStartNanos; + private final long minMeasurementNanos; + private long measuredFlowFileCount; + private long measurementWindowNanos; + private long completedInvocations; + private double inputQueueGrowth; + + private ConcurrencyComparison(final int originalConcurrentTasks, final int comparisonConcurrentTasks, + final double originalFlowFilesPerSecond, final double baselineFlowFilesPerSecond, + final long comparisonStartNanos, final long minMeasurementNanos) { + this.originalConcurrentTasks = originalConcurrentTasks; + this.comparisonConcurrentTasks = comparisonConcurrentTasks; + this.originalFlowFilesPerSecond = originalFlowFilesPerSecond; + this.baselineFlowFilesPerSecond = baselineFlowFilesPerSecond; + this.comparisonStartNanos = comparisonStartNanos; + this.minMeasurementNanos = minMeasurementNanos; + } + + private void addMeasurements(final ProcessorSchedulingSnapshot processorMeasurements) { + measuredFlowFileCount += processorMeasurements.committedFlowFiles(); + measurementWindowNanos += processorMeasurements.measurementWindowNanos(); + completedInvocations += processorMeasurements.completedInvocations(); + inputQueueGrowth += processorMeasurements.inputQueueGrowth(); + } + + private boolean isIncreasingConcurrentTasks() { + return comparisonConcurrentTasks > originalConcurrentTasks; + } + + private boolean hasMinimumMeasurements() { + return measurementWindowNanos >= minMeasurementNanos && (measuredFlowFileCount > 0L || completedInvocations > 0L); + } + + private boolean hasExpired(final long timestampNanos) { + return timestampNanos - comparisonStartNanos >= MAX_COMPARISON_NANOS; + } + + private double getMeasuredFlowFilesPerSecond() { + return measuredFlowFileCount * (double) TimeUnit.SECONDS.toNanos(1) / Math.max(1L, measurementWindowNanos); + } + + private boolean hasIncreasedThroughput(final double tolerance) { + return getMeasuredFlowFilesPerSecond() > originalFlowFilesPerSecond * (1D + tolerance); + } + + private boolean hasMaintainedThroughput(final double tolerance) { + return getMeasuredFlowFilesPerSecond() >= originalFlowFilesPerSecond * (1D - tolerance); + } + + private boolean hasInputQueueGrown() { + return inputQueueGrowth > 0D; + } + + private double getOriginalFlowFilesPerSecond() { + return originalFlowFilesPerSecond; + } + + private double getBaselineFlowFilesPerSecond() { + return baselineFlowFilesPerSecond; + } + + private int getOriginalConcurrentTasks() { + return originalConcurrentTasks; + } + + private int getComparisonConcurrentTasks() { + return comparisonConcurrentTasks; + } + + private long getMeasuredFlowFileCount() { + return measuredFlowFileCount; + } + + private long getMeasurementWindowNanos() { + return measurementWindowNanos; + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingMetrics.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingMetrics.java new file mode 100644 index 000000000000..3bacd489f64c --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingMetrics.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import com.sun.management.OperatingSystemMXBean; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.util.List; +import java.util.function.LongSupplier; + +public class SystemSchedulingMetrics { + private static final double CURRENT_CPU_LOAD_WEIGHT = 0.25D; + private static final double CPU_MAX_THRESHOLD = 0.90D; + private static final double CPU_INCREASE_RESUME_THRESHOLD = 0.80D; + private static final double GARBAGE_COLLECTION_MAX_THRESHOLD = 0.10D; + private static final int CPU_LOAD_READINGS_REQUIRED_TO_RESUME_INCREASES = 2; + private static final int GARBAGE_COLLECTION_READINGS_REQUIRED_TO_STOP_INCREASES = 2; + + private final OperatingSystemMXBean operatingSystemMXBean; + private final List garbageCollectorMXBeans; + private final LongSupplier nanoTimeSupplier; + + private double averageCpuLoad = -1D; + private long previousSampleNanos; + private long previousGarbageCollectionMillis; + private long previousProcessCpuNanos; + private int highGarbageCollectionReadings; + private int cpuReadingsBelowResumeThreshold; + private boolean cpuLoadAboveLimit; + private boolean garbageCollectionTimeAboveLimit; + + public SystemSchedulingMetrics() { + this(ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class), ManagementFactory.getGarbageCollectorMXBeans(), System::nanoTime); + } + + SystemSchedulingMetrics(final OperatingSystemMXBean operatingSystemMXBean, + final List garbageCollectorMXBeans, final LongSupplier nanoTimeSupplier) { + this.operatingSystemMXBean = operatingSystemMXBean; + // Collectors such as ZGC and Shenandoah report collection cycles that run alongside the application in beans named + // with a "Cycles" suffix, and report the time that the application is paused in separate beans. Only pauses are counted. + this.garbageCollectorMXBeans = garbageCollectorMXBeans.stream() + .filter(garbageCollectorMXBean -> !garbageCollectorMXBean.getName().endsWith(" Cycles")) + .toList(); + this.nanoTimeSupplier = nanoTimeSupplier; + previousSampleNanos = nanoTimeSupplier.getAsLong(); + previousGarbageCollectionMillis = getGarbageCollectionMillis(); + previousProcessCpuNanos = getProcessCpuTime(); + } + + public SystemSchedulingSnapshot captureSnapshot(final int maxGlobalConcurrentTasks, + final double fractionOfSamplesWithAllGlobalTaskSlotsUsed, + final double fractionOfGlobalTaskTimeSpentWaiting) { + final long nowNanos = nanoTimeSupplier.getAsLong(); + final long garbageCollectionMillis = getGarbageCollectionMillis(); + final long elapsedNanos = Math.max(1L, nowNanos - previousSampleNanos); + final long garbageCollectionDeltaMillis = Math.max(0L, garbageCollectionMillis - previousGarbageCollectionMillis); + final double fractionOfTimeSpentOnGarbageCollection = garbageCollectionDeltaMillis / (elapsedNanos / 1_000_000D); + + previousSampleNanos = nowNanos; + previousGarbageCollectionMillis = garbageCollectionMillis; + + updateGarbageCollectionState(fractionOfTimeSpentOnGarbageCollection); + + final double cpuLoad = getCpuLoad(elapsedNanos); + final boolean cpuLoadAvailable = cpuLoad >= 0D; + updateCpuLoadState(cpuLoad); + + return SystemSchedulingSnapshot.createBuilder() + .setCpuLoadAvailable(cpuLoadAvailable) + .setCpuLoad(cpuLoad) + .setAverageCpuLoad(averageCpuLoad) + .setFractionOfTimeSpentOnGarbageCollection(fractionOfTimeSpentOnGarbageCollection) + .setMaxGlobalConcurrentTasks(maxGlobalConcurrentTasks) + .setFractionOfSamplesWithAllGlobalTaskSlotsUsed(fractionOfSamplesWithAllGlobalTaskSlotsUsed) + .setFractionOfGlobalTaskTimeSpentWaiting(fractionOfGlobalTaskTimeSpentWaiting) + .setCpuLoadAboveLimit(cpuLoadAboveLimit) + .setGarbageCollectionTimeAboveLimit(garbageCollectionTimeAboveLimit) + .build(); + } + + private void updateGarbageCollectionState(final double fractionOfTimeSpentOnGarbageCollection) { + if (fractionOfTimeSpentOnGarbageCollection >= GARBAGE_COLLECTION_MAX_THRESHOLD) { + highGarbageCollectionReadings++; + } else { + highGarbageCollectionReadings = 0; + garbageCollectionTimeAboveLimit = false; + } + + if (highGarbageCollectionReadings >= GARBAGE_COLLECTION_READINGS_REQUIRED_TO_STOP_INCREASES) { + garbageCollectionTimeAboveLimit = true; + } + } + + private void updateCpuLoadState(final double cpuLoad) { + if (cpuLoad < 0D) { + averageCpuLoad = -1D; + cpuLoadAboveLimit = false; + cpuReadingsBelowResumeThreshold = 0; + return; + } + + // Retain 75% of the prior average and use 25% of the current reading so that one short spike does not stop concurrency increases. + averageCpuLoad = averageCpuLoad < 0D ? cpuLoad : CURRENT_CPU_LOAD_WEIGHT * cpuLoad + (1D - CURRENT_CPU_LOAD_WEIGHT) * averageCpuLoad; + if (averageCpuLoad >= CPU_MAX_THRESHOLD) { + cpuLoadAboveLimit = true; + cpuReadingsBelowResumeThreshold = 0; + } else if (averageCpuLoad < CPU_INCREASE_RESUME_THRESHOLD) { + cpuReadingsBelowResumeThreshold++; + if (cpuReadingsBelowResumeThreshold >= CPU_LOAD_READINGS_REQUIRED_TO_RESUME_INCREASES) { + cpuLoadAboveLimit = false; + } + } else { + cpuReadingsBelowResumeThreshold = 0; + } + } + + private double getCpuLoad(final long elapsedNanos) { + if (operatingSystemMXBean == null) { + return -1D; + } + + final double cpuLoad = Math.max(operatingSystemMXBean.getCpuLoad(), operatingSystemMXBean.getProcessCpuLoad()); + if (cpuLoad >= 0D) { + previousProcessCpuNanos = operatingSystemMXBean.getProcessCpuTime(); + return cpuLoad; + } + + final long processCpuNanos = operatingSystemMXBean.getProcessCpuTime(); + final long processCpuDeltaNanos = previousProcessCpuNanos < 0L ? -1L : processCpuNanos - previousProcessCpuNanos; + previousProcessCpuNanos = processCpuNanos; + if (processCpuDeltaNanos >= 0L) { + return Math.min(1D, processCpuDeltaNanos / (double) (elapsedNanos * operatingSystemMXBean.getAvailableProcessors())); + } + + return -1D; + } + + private long getProcessCpuTime() { + return operatingSystemMXBean == null ? -1L : operatingSystemMXBean.getProcessCpuTime(); + } + + private long getGarbageCollectionMillis() { + long totalMillis = 0L; + for (final GarbageCollectorMXBean garbageCollectorMXBean : garbageCollectorMXBeans) { + final long collectionTime = garbageCollectorMXBean.getCollectionTime(); + if (collectionTime > 0L) { + totalMillis += collectionTime; + } + } + + return totalMillis; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingSnapshot.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingSnapshot.java new file mode 100644 index 000000000000..a3bcae852263 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingSnapshot.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +public final class SystemSchedulingSnapshot { + private final boolean cpuLoadAvailable; + private final double cpuLoad; + private final double averageCpuLoad; + private final double fractionOfTimeSpentOnGarbageCollection; + private final int maxGlobalConcurrentTasks; + private final double fractionOfSamplesWithAllGlobalTaskSlotsUsed; + private final double fractionOfGlobalTaskTimeSpentWaiting; + private final boolean cpuLoadAboveLimit; + private final boolean garbageCollectionTimeAboveLimit; + + private SystemSchedulingSnapshot(final Builder builder) { + cpuLoadAvailable = builder.cpuLoadAvailable; + cpuLoad = builder.cpuLoad; + averageCpuLoad = builder.averageCpuLoad; + fractionOfTimeSpentOnGarbageCollection = builder.fractionOfTimeSpentOnGarbageCollection; + maxGlobalConcurrentTasks = builder.maxGlobalConcurrentTasks; + fractionOfSamplesWithAllGlobalTaskSlotsUsed = builder.fractionOfSamplesWithAllGlobalTaskSlotsUsed; + fractionOfGlobalTaskTimeSpentWaiting = builder.fractionOfGlobalTaskTimeSpentWaiting; + cpuLoadAboveLimit = builder.cpuLoadAboveLimit; + garbageCollectionTimeAboveLimit = builder.garbageCollectionTimeAboveLimit; + } + + public static Builder createBuilder() { + return new Builder(); + } + + public boolean cpuLoadAvailable() { + return cpuLoadAvailable; + } + + public double cpuLoad() { + return cpuLoad; + } + + public double averageCpuLoad() { + return averageCpuLoad; + } + + public double fractionOfTimeSpentOnGarbageCollection() { + return fractionOfTimeSpentOnGarbageCollection; + } + + public int maxGlobalConcurrentTasks() { + return maxGlobalConcurrentTasks; + } + + public double fractionOfSamplesWithAllGlobalTaskSlotsUsed() { + return fractionOfSamplesWithAllGlobalTaskSlotsUsed; + } + + public double fractionOfGlobalTaskTimeSpentWaiting() { + return fractionOfGlobalTaskTimeSpentWaiting; + } + + public boolean cpuLoadAboveLimit() { + return cpuLoadAboveLimit; + } + + public boolean garbageCollectionTimeAboveLimit() { + return garbageCollectionTimeAboveLimit; + } + + public static final class Builder { + private boolean cpuLoadAvailable; + private double cpuLoad; + private double averageCpuLoad; + private double fractionOfTimeSpentOnGarbageCollection; + private int maxGlobalConcurrentTasks; + private double fractionOfSamplesWithAllGlobalTaskSlotsUsed; + private double fractionOfGlobalTaskTimeSpentWaiting; + private boolean cpuLoadAboveLimit; + private boolean garbageCollectionTimeAboveLimit; + + private Builder() { + } + + public Builder setCpuLoadAvailable(final boolean cpuLoadAvailable) { + this.cpuLoadAvailable = cpuLoadAvailable; + return this; + } + + public Builder setCpuLoad(final double cpuLoad) { + this.cpuLoad = cpuLoad; + return this; + } + + public Builder setAverageCpuLoad(final double averageCpuLoad) { + this.averageCpuLoad = averageCpuLoad; + return this; + } + + public Builder setFractionOfTimeSpentOnGarbageCollection(final double fractionOfTimeSpentOnGarbageCollection) { + this.fractionOfTimeSpentOnGarbageCollection = fractionOfTimeSpentOnGarbageCollection; + return this; + } + + public Builder setMaxGlobalConcurrentTasks(final int maxGlobalConcurrentTasks) { + this.maxGlobalConcurrentTasks = maxGlobalConcurrentTasks; + return this; + } + + public Builder setFractionOfSamplesWithAllGlobalTaskSlotsUsed(final double fractionOfSamplesWithAllGlobalTaskSlotsUsed) { + this.fractionOfSamplesWithAllGlobalTaskSlotsUsed = fractionOfSamplesWithAllGlobalTaskSlotsUsed; + return this; + } + + public Builder setFractionOfGlobalTaskTimeSpentWaiting(final double fractionOfGlobalTaskTimeSpentWaiting) { + this.fractionOfGlobalTaskTimeSpentWaiting = fractionOfGlobalTaskTimeSpentWaiting; + return this; + } + + public Builder setCpuLoadAboveLimit(final boolean cpuLoadAboveLimit) { + this.cpuLoadAboveLimit = cpuLoadAboveLimit; + return this; + } + + public Builder setGarbageCollectionTimeAboveLimit(final boolean garbageCollectionTimeAboveLimit) { + this.garbageCollectionTimeAboveLimit = garbageCollectionTimeAboveLimit; + return this; + } + + public SystemSchedulingSnapshot build() { + return new SystemSchedulingSnapshot(this); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ConnectableTask.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ConnectableTask.java index 7c94e8a61630..31e94510b026 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ConnectableTask.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ConnectableTask.java @@ -38,6 +38,8 @@ import org.apache.nifi.controller.scheduling.LifecycleState; import org.apache.nifi.controller.scheduling.RepositoryContextFactory; import org.apache.nifi.controller.scheduling.SchedulingAgent; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; +import org.apache.nifi.controller.status.FlowFileAvailability; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.logging.StandardLoggingContext; import org.apache.nifi.nar.NarCloseable; @@ -66,9 +68,9 @@ public class ConnectableTask { private static final Logger logger = LoggerFactory.getLogger(ConnectableTask.class); private static final BooleanSupplier SCHEDULING_GENERATION_ALWAYS_ACTIVE = () -> true; - private static final InvocationResult NOT_PRIMARY_NODE_YIELD_RESULT = InvocationResult.yield("This node is not the primary node"); - private static final InvocationResult NO_WORK_YIELD_RESULT = InvocationResult.yield("No work to do"); - private static final InvocationResult BACKPRESSURE_YIELD_RESULT = InvocationResult.yield("Backpressure Applied"); + private static final InvocationResult NOT_PRIMARY_NODE_YIELD_RESULT = InvocationResult.yield(InvocationOutcome.NOT_PRIMARY, "This node is not the primary node"); + private static final InvocationResult NO_WORK_YIELD_RESULT = InvocationResult.yield(InvocationOutcome.NO_INPUT, "No work to do"); + private static final InvocationResult BACKPRESSURE_YIELD_RESULT = InvocationResult.yield(InvocationOutcome.BACKPRESSURED, "Backpressure Applied"); private final SchedulingAgent schedulingAgent; private final Connectable connectable; @@ -105,8 +107,10 @@ public ConnectableTask(final SchedulingAgent schedulingAgent, final Connectable } final StateManager stateManager = new TaskTerminationAwareStateManager(baseStateManager, lifecycleState::isTerminated); if (connectable instanceof final ProcessorNode processorNode) { - processContext = new StandardProcessContext( - processorNode, flowController.getControllerServiceProvider(), stateManager, lifecycleState::isTerminated, flowController); + processContext = StandardProcessContext.createBuilder( + processorNode, flowController.getControllerServiceProvider(), stateManager, lifecycleState::isTerminated, flowController) + .setMaxConcurrentTasks(schedulingAgent.getProcessContextConcurrencyLimit(connectable)) + .build(); } else { processContext = new ConnectableProcessContext(connectable, stateManager); } @@ -172,16 +176,53 @@ private boolean isBackPressureEngaged() { return false; } + private boolean isInputPenalized() { + for (final Connection connection : connectable.getIncomingConnections()) { + if (connection.getFlowFileQueue().getFlowFileAvailability() == FlowFileAvailability.HEAD_OF_QUEUE_PENALIZED) { + return true; + } + } + + return false; + } + public InvocationResult invoke() { + return invoke(connectable.getRunDuration(TimeUnit.NANOSECONDS), () -> true, InvocationObserver.NO_OP); + } + + public InvocationResult invoke(final long effectiveRunDurationNanos, final BooleanSupplier continueRunning, final InvocationObserver observer) { + final InvocationResult result = invokeInternal(effectiveRunDurationNanos, continueRunning, observer); + try { + observer.onInvocationCompleted(result); + } catch (final Throwable observerFailure) { + logger.warn("Invocation observer failed after invoking {}", connectable, observerFailure); + } + + return result; + } + + public boolean isReady() { + return getReadinessResult() == null; + } + + public InvocationOutcome getReadinessOutcome() { + final InvocationResult readinessResult = getReadinessResult(); + return readinessResult == null ? null : readinessResult.getOutcome(); + } + + public boolean hasLocallyConsumableInput() { + return Connectables.flowFilesQueued(connectable); + } + + private InvocationResult getReadinessResult() { if (lifecycleState.isTerminated()) { logger.debug("Will not trigger {} because task is terminated", connectable); - return InvocationResult.DO_NOT_YIELD; + return InvocationResult.completed(InvocationOutcome.STOPPED); } - // make sure processor is not yielded if (isYielded()) { logger.debug("Will not trigger {} because component is yielded", connectable); - return InvocationResult.DO_NOT_YIELD; + return InvocationResult.completed(InvocationOutcome.YIELDED); } // make sure that either we're not clustered or this processor runs on all nodes or that this is the primary node @@ -200,6 +241,9 @@ public InvocationResult invoke() { // Make sure processor has work to do. if (!isWorkToDo()) { logger.debug("Yielding {} because it has no work to do", connectable); + if (isInputPenalized()) { + return InvocationResult.yield(InvocationOutcome.PENALIZED_INPUT, "Input is penalized"); + } return NO_WORK_YIELD_RESULT; } @@ -211,20 +255,34 @@ public InvocationResult invoke() { } } + return null; + } + + private InvocationResult invokeInternal(final long effectiveRunDurationNanos, final BooleanSupplier continueRunning, final InvocationObserver observer) { + if (!continueRunning.getAsBoolean()) { + return InvocationResult.completed(InvocationOutcome.STOPPED); + } + + final InvocationResult readinessResult = getReadinessResult(); + if (readinessResult != null) { + return readinessResult; + } + logger.debug("Triggering {}", connectable); final TrackedStats stats = statsTracker.startTracking(); - final long batchNanos = connectable.getRunDuration(TimeUnit.NANOSECONDS); + final long batchNanos = effectiveRunDurationNanos; + final SessionSchedulingObserver sessionObserver = observer == InvocationObserver.NO_OP ? SessionSchedulingObserver.NO_OP : observer; final ProcessSessionFactory sessionFactory; final StandardProcessSession rawSession; final boolean batch; if (connectable.isSessionBatchingSupported() && batchNanos > 0L) { - rawSession = new StandardProcessSession(repositoryContext, lifecycleState::isTerminated, stats.getPerformanceTracker()); + rawSession = new StandardProcessSession(repositoryContext, lifecycleState::isTerminated, stats.getPerformanceTracker(), sessionObserver); sessionFactory = new BatchingSessionFactory(rawSession); batch = true; } else { rawSession = null; - sessionFactory = new StandardProcessSessionFactory(repositoryContext, lifecycleState::isTerminated, stats.getPerformanceTracker()); + sessionFactory = new StandardProcessSessionFactory(repositoryContext, lifecycleState::isTerminated, stats.getPerformanceTracker(), sessionObserver); batch = false; } @@ -246,22 +304,24 @@ public InvocationResult invoke() { final String originalThreadName = Thread.currentThread().getName(); try { try (final AutoCloseable ignored = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), connectable.getRunnableComponent().getClass(), connectable.getIdentifier())) { - boolean shouldRun = connectable.getScheduledState() == ScheduledState.RUNNING || connectable.getScheduledState() == ScheduledState.RUN_ONCE; + boolean shouldRun = (connectable.getScheduledState() == ScheduledState.RUNNING || connectable.getScheduledState() == ScheduledState.RUN_ONCE) + && continueRunning.getAsBoolean(); while (shouldRun) { invocationCount++; + observer.resetTriggerActivity(); connectable.onTrigger(processContext, activeSessionFactory); - if (!batch) { - return InvocationResult.DO_NOT_YIELD; + if (!batch || !observer.isTriggerActivityObserved()) { + return getCompletedResult(observer); } final long nanoTime = System.nanoTime(); if (nanoTime > finishNanos) { - return InvocationResult.DO_NOT_YIELD; + return getCompletedResult(observer); } if (nanoTime > finishIfBackpressureEngaged && isBackPressureEngaged()) { - return InvocationResult.DO_NOT_YIELD; + return getCompletedResult(observer); } if (connectable.getScheduledState() != ScheduledState.RUNNING) { @@ -274,6 +334,9 @@ public InvocationResult invoke() { if (isYielded()) { break; } + if (!continueRunning.getAsBoolean()) { + break; + } if (numRelationships > 0) { final int requiredNumberOfAvailableRelationships = connectable.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships; @@ -283,14 +346,17 @@ public InvocationResult invoke() { } catch (final TerminatedTaskException e) { final ComponentLog componentLog = getComponentLog(); componentLog.info("Processing terminated", e); + return InvocationResult.completed(InvocationOutcome.STOPPED); } catch (final ProcessException e) { final ComponentLog componentLog = getComponentLog(); componentLog.error("Processing failed", e); + return InvocationResult.completed(InvocationOutcome.FAILED); } catch (final Throwable e) { final ComponentLog componentLog = getComponentLog(); componentLog.error("Processing halted: yielding [{}]", schedulingAgent.getAdministrativeYieldDuration(), e); logger.warn("Processing halted: uncaught exception in Component [{}]", connectable.getRunnableComponent(), e); connectable.yield(schedulingAgent.getAdministrativeYieldDuration(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS); + return InvocationResult.completed(InvocationOutcome.FAILED); } } finally { try { @@ -317,7 +383,16 @@ public InvocationResult invoke() { } } - return InvocationResult.DO_NOT_YIELD; + return getCompletedResult(observer); + } + + private InvocationResult getCompletedResult(final InvocationObserver observer) { + try { + return InvocationResult.completed(observer.isActivityObserved() ? InvocationOutcome.INVOKED_WITH_ACTIVITY : InvocationOutcome.INVOKED_WITHOUT_ACTIVITY); + } catch (final Throwable observerFailure) { + logger.warn("Invocation observer failed while checking activity for {}", connectable, observerFailure); + return InvocationResult.completed(InvocationOutcome.INVOKED_WITHOUT_ACTIVITY); + } } private void updateEventRepo(final TrackedStats stats, final int invocationCount) throws IOException { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ExpireFlowFiles.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ExpireFlowFiles.java index d1523de6f7d7..07c78e200cd5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ExpireFlowFiles.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ExpireFlowFiles.java @@ -27,6 +27,7 @@ import org.apache.nifi.controller.repository.StandardProcessSessionFactory; import org.apache.nifi.controller.repository.metrics.NopPerformanceTracker; import org.apache.nifi.controller.scheduling.RepositoryContextFactory; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.util.FormatUtils; @@ -62,7 +63,7 @@ public void run() { private StandardProcessSession createSession(final Connectable connectable) { final RepositoryContext context = contextFactory.newProcessContext(connectable, new AtomicLong(0L)); - final StandardProcessSessionFactory sessionFactory = new StandardProcessSessionFactory(context, () -> false, new NopPerformanceTracker()); + final StandardProcessSessionFactory sessionFactory = new StandardProcessSessionFactory(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); return sessionFactory.createSession(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationObserver.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationObserver.java new file mode 100644 index 000000000000..764da762b640 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationObserver.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.tasks; + +import org.apache.nifi.controller.scheduling.CommittedSchedulingWork; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; + +public interface InvocationObserver extends SessionSchedulingObserver { + + InvocationObserver NO_OP = new InvocationObserver() { + @Override + public boolean isActivityObserved() { + return false; + } + + @Override + public void onInvocationCompleted(final InvocationResult result) { + } + + @Override + public void onActivity() { + } + + @Override + public void onCommit(final CommittedSchedulingWork work) { + } + + @Override + public void onCommitFailure() { + } + }; + + boolean isActivityObserved(); + + default void resetTriggerActivity() { + } + + default boolean isTriggerActivityObserved() { + // Invocations without activity tracking retain their configured batching behavior. + return true; + } + + void onInvocationCompleted(InvocationResult result); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationOutcome.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationOutcome.java new file mode 100644 index 000000000000..63eb02ba7db7 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationOutcome.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.tasks; + +public enum InvocationOutcome { + INVOKED_WITH_ACTIVITY, + INVOKED_WITHOUT_ACTIVITY, + NO_INPUT, + PENALIZED_INPUT, + BACKPRESSURED, + YIELDED, + NOT_PRIMARY, + STOPPED, + FAILED +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationResult.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationResult.java index 0d8f51f8d105..d8844546e57f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationResult.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/InvocationResult.java @@ -22,29 +22,40 @@ public interface InvocationResult { String getYieldExplanation(); - InvocationResult DO_NOT_YIELD = new InvocationResult() { - @Override - public boolean isYield() { - return false; - } - - @Override - public String getYieldExplanation() { - return null; - } - }; + default InvocationOutcome getOutcome() { + return isYield() ? InvocationOutcome.YIELDED : InvocationOutcome.INVOKED_WITH_ACTIVITY; + } + + InvocationResult DO_NOT_YIELD = result(InvocationOutcome.INVOKED_WITH_ACTIVITY, false, null); static InvocationResult yield(final String explanation) { + return result(InvocationOutcome.YIELDED, true, explanation); + } + + static InvocationResult yield(final InvocationOutcome outcome, final String explanation) { + return result(outcome, true, explanation); + } + + static InvocationResult completed(final InvocationOutcome outcome) { + return result(outcome, false, null); + } + + private static InvocationResult result(final InvocationOutcome outcome, final boolean yield, final String explanation) { return new InvocationResult() { @Override public boolean isYield() { - return true; + return yield; } @Override public String getYieldExplanation() { return explanation; } + + @Override + public InvocationOutcome getOutcome() { + return outcome; + } }; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/FlowControllerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/FlowControllerTest.java new file mode 100644 index 000000000000..ae8b9cb615b1 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/FlowControllerTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller; + +import org.apache.nifi.controller.scheduling.auto.AutoSchedulingDiagnostics; +import org.apache.nifi.scheduling.SchedulingStrategy; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class FlowControllerTest { + + @Test + void testPlatformAutomaticDiagnosticsUseProcessorActiveCount() { + final FlowController flowController = mock(FlowController.class, CALLS_REAL_METHODS); + final ProcessorNode processorNode = mock(ProcessorNode.class); + when(processorNode.getSchedulingStrategy()).thenReturn(SchedulingStrategy.AUTO); + when(processorNode.getActiveThreadCount()).thenReturn(7); + + final AutoSchedulingDiagnostics diagnostics = flowController.getAutoSchedulingDiagnostics(processorNode); + assertEquals(7, diagnostics.activeProcessorInvocations()); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowSnippetTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowSnippetTest.java new file mode 100644 index 000000000000..3a8cc6741cce --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowSnippetTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller; + +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; +import org.apache.nifi.bundle.Bundle; +import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.bundle.BundleDetails; +import org.apache.nifi.nar.ExtensionDefinition; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.processor.Processor; +import org.apache.nifi.scheduling.SchedulingStrategy; +import org.apache.nifi.web.api.dto.BundleDTO; +import org.apache.nifi.web.api.dto.FlowSnippetDTO; +import org.apache.nifi.web.api.dto.ProcessorConfigDTO; +import org.apache.nifi.web.api.dto.ProcessorDTO; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StandardFlowSnippetTest { + + @Test + void testUnsupportedAutomaticProcessorRejectedBeforeInstantiation() { + final BundleCoordinate coordinate = new BundleCoordinate("group", "artifact", "version"); + final BundleDTO bundle = new BundleDTO(); + bundle.setGroup(coordinate.getGroup()); + bundle.setArtifact(coordinate.getId()); + bundle.setVersion(coordinate.getVersion()); + + final ProcessorConfigDTO config = new ProcessorConfigDTO(); + config.setSchedulingStrategy(SchedulingStrategy.AUTO.name()); + final ProcessorDTO processor = new ProcessorDTO(); + processor.setId("processor-id"); + processor.setName("Unsupported Processor"); + processor.setType(UnsupportedAutomaticProcessor.class.getName()); + processor.setBundle(bundle); + processor.setConfig(config); + final FlowSnippetDTO snippet = new FlowSnippetDTO(); + snippet.setProcessors(Set.of(processor)); + + final ExtensionDefinition definition = mock(ExtensionDefinition.class); + when(definition.getImplementationClassName()).thenReturn(processor.getType()); + final BundleDetails bundleDetails = mock(BundleDetails.class); + when(bundleDetails.getCoordinate()).thenReturn(coordinate); + final Bundle installedBundle = mock(Bundle.class); + when(installedBundle.getBundleDetails()).thenReturn(bundleDetails); + final ExtensionManager extensionManager = mock(ExtensionManager.class); + when(extensionManager.getExtensions(Processor.class)).thenReturn(Set.of(definition)); + when(extensionManager.getBundles(processor.getType())).thenReturn(List.of(installedBundle)); + + final StandardFlowSnippet flowSnippet = new StandardFlowSnippet(snippet, extensionManager); + final IllegalStateException unresolvedException = assertThrows(IllegalStateException.class, flowSnippet::verifyComponentTypesInSnippet); + assertEquals("Processor Unsupported Processor [processor-id] cannot use scheduling strategy AUTO because its automatic scheduling capability could not be resolved", + unresolvedException.getMessage()); + + when(extensionManager.getTempComponent(processor.getType(), coordinate)).thenReturn(mock(UnsupportedAutomaticProcessor.class)); + final IllegalStateException exception = assertThrows(IllegalStateException.class, flowSnippet::verifyComponentTypesInSnippet); + + assertEquals("Processor Unsupported Processor [processor-id] cannot use scheduling strategy AUTO because its implementation disables automatic scheduling", + exception.getMessage()); + } + + @AllowsAutoScheduling(false) + private abstract static class UnsupportedAutomaticProcessor implements Processor { + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java index e1aac575a0a9..5319875a3f65 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestStandardFlowFileQueue.java @@ -24,6 +24,7 @@ import org.apache.nifi.controller.queue.DropFlowFileStatus; import org.apache.nifi.controller.queue.ListFlowFileState; import org.apache.nifi.controller.queue.ListFlowFileStatus; +import org.apache.nifi.controller.queue.QueueSchedulingRegistration; import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.controller.queue.StandardFlowFileQueue; import org.apache.nifi.controller.repository.FlowFileRecord; @@ -42,6 +43,7 @@ import org.mockito.Mockito; import org.mockito.stubbing.Answer; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -49,6 +51,10 @@ import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -170,6 +176,58 @@ public void testBackPressure() { assertFalse(queue.isActiveQueueEmpty()); } + @Test + public void testSchedulingListenerNotificationsAndRemoval() { + final AtomicInteger notificationCount = new AtomicInteger(); + final QueueSchedulingRegistration registration = queue.addSchedulingListener(notificationCount::incrementAndGet); + + final FlowFileRecord flowFile = new MockFlowFileRecord(); + queue.put(flowFile); + assertEquals(1, notificationCount.get()); + + final Set expiredRecords = new HashSet<>(); + final FlowFileRecord polled = queue.poll(expiredRecords); + assertEquals(flowFile, polled); + assertEquals(2, notificationCount.get()); + assertEquals(new QueueSize(0, 0L), queue.getLocalQueueSize()); + + queue.acknowledge(polled); + assertEquals(3, notificationCount.get()); + + registration.close(); + registration.close(); + queue.put(new MockFlowFileRecord()); + assertEquals(3, notificationCount.get()); + } + + @Test + public void testSelectiveDropNotifiesSchedulingListenerOutsideQueueLock() throws IOException { + queue.put(new MockFlowFileRecord()); + final AtomicBoolean listenerInvoked = new AtomicBoolean(); + final AtomicBoolean concurrentMutationCompleted = new AtomicBoolean(); + final AtomicBoolean notificationOutsideLock = new AtomicBoolean(); + queue.addSchedulingListener(() -> { + if (!listenerInvoked.compareAndSet(false, true)) { + return; + } + + Thread.ofVirtual().start(() -> { + queue.put(new MockFlowFileRecord()); + concurrentMutationCompleted.set(true); + }); + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1L); + while (!concurrentMutationCompleted.get() && System.nanoTime() < deadline) { + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1L)); + } + notificationOutsideLock.set(concurrentMutationCompleted.get()); + }); + + final DropFlowFileSummary summary = queue.dropFlowFiles(flowFile -> true); + + assertEquals(1L, summary.getDroppedCount()); + assertTrue(notificationOutsideLock.get()); + } + @Test public void testBackPressureAfterPollFilter() throws InterruptedException { queue.setBackPressureObjectThreshold(10); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSocketLoadBalancedFlowFileQueue.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSocketLoadBalancedFlowFileQueue.java index 6c683ec9d5b3..65c07bb98582 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSocketLoadBalancedFlowFileQueue.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/queue/clustered/TestSocketLoadBalancedFlowFileQueue.java @@ -27,6 +27,7 @@ import org.apache.nifi.controller.MockSwapManager; import org.apache.nifi.controller.ProcessScheduler; import org.apache.nifi.controller.queue.FlowFileQueueSnapshot; +import org.apache.nifi.controller.queue.IllegalClusterStateException; import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.controller.queue.clustered.client.async.AsyncLoadBalanceClientRegistry; import org.apache.nifi.controller.queue.clustered.partition.FlowFilePartitioner; @@ -64,8 +65,9 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -197,6 +199,28 @@ public void testFlowFileAvailability() { assertTrue(queue.isEmpty()); } + @Test + public void testSchedulingNotificationsForPeerReceiptTransferAndAbort() throws IllegalClusterStateException { + when(clusterCoordinator.isConnected()).thenReturn(true); + queue.setFlowFilePartitioner(new StaticFlowFilePartitioner(determineLocalPartitionIndex())); + final AtomicInteger notifications = new AtomicInteger(); + queue.addSchedulingListener(notifications::incrementAndGet); + + final MockFlowFileRecord received = new MockFlowFileRecord(10L); + queue.receiveFromPeer(List.of(received)); + assertEquals(1, notifications.getAndSet(0)); + assertEquals(new QueueSize(1, 10L), queue.getLocalQueueSize()); + + queue.onTransfer(List.of(received)); + assertEquals(1, notifications.getAndSet(0)); + + final MockFlowFileRecord aborted = new MockFlowFileRecord(20L); + queue.put(aborted); + notifications.set(0); + queue.onAbort(List.of(aborted)); + assertEquals(1, notifications.get()); + } + @Test public void testPriorities() { final FlowFilePrioritizer iValuePrioritizer = (o1, o2) -> { @@ -338,11 +362,13 @@ public void testIsEmptyWhenFlowFileInRemotePartition() { assertFalse(queue.isEmpty()); assertTrue(queue.isActiveQueueEmpty()); assertEquals(new QueueSize(1, 0L), queue.size()); + assertEquals(new QueueSize(0, 0L), queue.getLocalQueueSize()); assertNull(queue.poll(new HashSet<>())); assertFalse(queue.isEmpty()); assertTrue(queue.isActiveQueueEmpty()); assertEquals(new QueueSize(1, 0L), queue.size()); + assertEquals(new QueueSize(0, 0L), queue.getLocalQueueSize()); } @Test @@ -360,10 +386,12 @@ public void testIsEmptyWhenFlowFileInLocalPartition() { assertFalse(queue.isEmpty()); assertFalse(queue.isActiveQueueEmpty()); assertEquals(new QueueSize(1, 0L), queue.size()); + assertEquals(new QueueSize(1, 0L), queue.getLocalQueueSize()); // Ensure that we get the same FlowFile back. This will not decrement // the queue size, only acknowledging the FlowFile will do that. assertSame(flowFile, queue.poll(new HashSet<>())); + assertEquals(new QueueSize(0, 0L), queue.getLocalQueueSize()); assertFalse(queue.isEmpty()); assertTrue(queue.isActiveQueueEmpty()); assertEquals(new QueueSize(1, 0L), queue.size()); @@ -801,7 +829,7 @@ public void testGetQueueSnapshotIncludesRebalancingPartition() { private void assertPartitionSizes(final int[] expectedSizes) { final int[] partitionSizes = new int[queue.getPartitionCount()]; while (!Arrays.equals(expectedSizes, partitionSizes)) { - assertDoesNotThrow(() -> Thread.sleep(10L)); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(10L)); for (int i = 0; i < partitionSizes.length; i++) { partitionSizes[i] = queue.getPartition(i).size().getObjectCount(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/BatchingSessionFactoryTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/BatchingSessionFactoryTest.java index 99b0a9e88547..66492afb646f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/BatchingSessionFactoryTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/BatchingSessionFactoryTest.java @@ -20,6 +20,7 @@ import org.apache.nifi.controller.lifecycle.TaskTermination; import org.apache.nifi.controller.metrics.GaugeRecord; import org.apache.nifi.controller.repository.metrics.NopPerformanceTracker; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; import org.apache.nifi.processor.ProcessSession; import org.apache.nifi.processor.metrics.CommitTiming; import org.junit.jupiter.api.BeforeEach; @@ -71,7 +72,7 @@ class BatchingSessionFactoryTest { void setFactory() { when(repositoryContext.getConnectable()).thenReturn(connectable); when(connectable.getIdentifier()).thenReturn(CONNECTABLE_ID); - final StandardProcessSession standardProcessSession = new StandardProcessSession(repositoryContext, taskTermination, new NopPerformanceTracker()); + final StandardProcessSession standardProcessSession = new StandardProcessSession(repositoryContext, taskTermination, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); factory = new BatchingSessionFactory(standardProcessSession); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/StandardProcessSessionIT.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/StandardProcessSessionIT.java index 6baab772b824..f5ca19ccb094 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/StandardProcessSessionIT.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/StandardProcessSessionIT.java @@ -41,6 +41,7 @@ import org.apache.nifi.controller.repository.claim.StandardResourceClaimManager; import org.apache.nifi.controller.repository.metrics.NopPerformanceTracker; import org.apache.nifi.controller.repository.metrics.RingBufferEventRepository; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; import org.apache.nifi.events.EventReporter; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.flowfile.attributes.CoreAttributes; @@ -234,7 +235,7 @@ public void setup() throws IOException { context = new StandardRepositoryContext(connectable, new AtomicLong(0L), contentRepo, flowFileRepo, flowFileEventRepository, counterRepository, componentMetricReporter, provenanceRepo, stateManager); - session = new StandardProcessSession(context, () -> false, new NopPerformanceTracker()); + session = new StandardProcessSession(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); } private Connection createConnection() { @@ -338,7 +339,7 @@ public void testFlowFileHandlingExceptionThrownIfMigratingChildNotParent() { children.add(child); } - final ProcessSession secondSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker()); + final ProcessSession secondSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); assertThrows(FlowFileHandlingException.class, () -> session.migrate(secondSession, children), "Expected a FlowFileHandlingException to be thrown because a child FlowFile was migrated while its parent was not"); @@ -373,7 +374,7 @@ public void testCloneForkChildMigrateCommit() throws IOException { FlowFile flowFile = session.get(); assertNotNull(flowFile); - final ProcessSession secondSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker()); + final ProcessSession secondSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); FlowFile clone = session.clone(flowFile); session.migrate(secondSession, Collections.singletonList(clone)); @@ -1667,7 +1668,7 @@ public void testAppendToFlowFileWhereResourceClaimHasMultipleContentClaims() thr session.transfer(ffb, relationship); session.commit(); - final ProcessSession newSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker()); + final ProcessSession newSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); FlowFile toUpdate = newSession.get(); newSession.append(toUpdate, out -> out.write('C')); @@ -2211,7 +2212,7 @@ public void testMigrateWithAppendableStream() throws IOException { flowFile = session.append(flowFile, out -> out.write("1".getBytes())); flowFile = session.append(flowFile, out -> out.write("2".getBytes())); - final StandardProcessSession newSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker()); + final StandardProcessSession newSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); assertTrue(session.isFlowFileKnown(flowFile)); assertFalse(newSession.isFlowFileKnown(flowFile)); @@ -2241,7 +2242,7 @@ public void testMigrateToDelegatingSessionWrapper() { FlowFile flowFile = session.create(); flowFile = session.write(flowFile, out -> out.write("contents".getBytes(StandardCharsets.UTF_8))); - final StandardProcessSession newSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker()); + final StandardProcessSession newSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); // Simulate the framework path that wraps a Session through WeakHashMapProcessSessionFactory: the // Processor receives the wrapper, and other framework code that holds the underlying StandardProcessSession @@ -2269,7 +2270,7 @@ public void testMigrateAfterTransferToAutoTerminatedRelationship() { FlowFile flowFile = session.create(); flowFile = session.write(flowFile, out -> out.write("Hello".getBytes(StandardCharsets.UTF_8))); - final StandardProcessSession newSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker()); + final StandardProcessSession newSession = new StandardProcessSession(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); when(connectable.getConnections(any(Relationship.class))).thenReturn(Collections.emptySet()); when(connectable.isAutoTerminated(any(Relationship.class))).thenReturn(true); @@ -2388,7 +2389,7 @@ public void testContentClaimCreationContextLossToleranceMatchesComponent() { final StandardRepositoryContext lossTolerantContext = new StandardRepositoryContext(connectable, new AtomicLong(0L), contentRepo, flowFileRepo, flowFileEventRepository, counterRepository, componentMetricReporter, provenanceRepo, stateManager); - final StandardProcessSession lossTolerantSession = new StandardProcessSession(lossTolerantContext, () -> false, new NopPerformanceTracker()); + final StandardProcessSession lossTolerantSession = new StandardProcessSession(lossTolerantContext, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); try { FlowFile flowFile = lossTolerantSession.create(); @@ -3188,7 +3189,7 @@ public StandardProcessSession createSessionForRetry(final Connectable connectabl componentMetricReporter, provenanceRepo, stateManager); - return new StandardProcessSession(context, () -> false, new NopPerformanceTracker()); + return new StandardProcessSession(context, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgentTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgentTest.java index 47428da4b8ea..2bae49d829de 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgentTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgentTest.java @@ -19,12 +19,22 @@ import org.apache.nifi.components.state.StateManager; import org.apache.nifi.components.state.StateManagerProvider; import org.apache.nifi.connectable.Connectable; +import org.apache.nifi.connectable.Connection; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.GarbageCollectionLog; +import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.ScheduledState; +import org.apache.nifi.controller.queue.FlowFileQueue; +import org.apache.nifi.controller.queue.QueueSchedulingListener; +import org.apache.nifi.controller.queue.QueueSchedulingRegistration; +import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.controller.repository.FlowFileEventRepository; import org.apache.nifi.controller.repository.RepositoryContext; +import org.apache.nifi.controller.scheduling.auto.AutoSchedulingDiagnostics; +import org.apache.nifi.controller.scheduling.auto.SystemSchedulingMetrics; +import org.apache.nifi.controller.scheduling.auto.SystemSchedulingSnapshot; +import org.apache.nifi.controller.status.FlowFileAvailability; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarThreadContextClassLoader; @@ -42,7 +52,9 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import java.time.Instant; import java.util.Collections; +import java.util.List; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -50,6 +62,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.IntSupplier; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -57,9 +70,13 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -96,7 +113,15 @@ class VirtualThreadSchedulingAgentTest { @BeforeEach void setUp() { when(nifiProperties.getBoredYieldDuration()).thenReturn("10 millis"); - agent = new VirtualThreadSchedulingAgent(flowController, contextFactory, nifiProperties, MAX_THREADS); + when(nifiProperties.getProcessorAutoMaxConcurrentTasks()).thenReturn(12); + final SystemSchedulingMetrics systemSchedulingMetrics = mock(SystemSchedulingMetrics.class); + when(systemSchedulingMetrics.captureSnapshot(anyInt(), anyDouble(), anyDouble())).thenAnswer(invocation -> SystemSchedulingSnapshot.createBuilder() + .setCpuLoadAvailable(true) + .setCpuLoad(0.25D) + .setAverageCpuLoad(0.25D) + .setMaxGlobalConcurrentTasks(invocation.getArgument(0, Integer.class)) + .build()); + agent = new VirtualThreadSchedulingAgent(flowController, contextFactory, nifiProperties, MAX_THREADS, 12, systemSchedulingMetrics); } @AfterEach @@ -121,6 +146,284 @@ void testIncrementMaxThreadCountAdjustsSemaphore() { assertThrows(IllegalStateException.class, () -> agent.incrementMaxThreadCount(-1000)); } + @Test + void testProcessContextConcurrencyLimit() { + final Connectable timerDriven = mock(Connectable.class); + when(timerDriven.getSchedulingStrategy()).thenReturn(SchedulingStrategy.TIMER_DRIVEN); + when(timerDriven.getMaxConcurrentTasks()).thenReturn(4); + assertEquals(4, agent.getProcessContextConcurrencyLimit(timerDriven)); + + final ProcessorNode automatic = mock(ProcessorNode.class); + when(automatic.getSchedulingStrategy()).thenReturn(SchedulingStrategy.AUTO); + assertEquals(12, agent.getProcessContextConcurrencyLimit(automatic)); + + when(automatic.isTriggeredSerially()).thenReturn(true); + assertEquals(1, agent.getProcessContextConcurrencyLimit(automatic)); + } + + @Test + void testAutomaticSchedulingBacksOffEmptySource() throws InterruptedException { + final AtomicInteger invocationCount = new AtomicInteger(); + final CountDownLatch firstInvocation = new CountDownLatch(1); + final Connectable connectable = createMockedConnectable(1, SchedulingStrategy.AUTO, invocationCount, firstInvocation); + final LifecycleState lifecycleState = new LifecycleState(COMPONENT_ID); + + scheduleConnectable(connectable, lifecycleState); + try { + assertTrue(firstInvocation.await(2, TimeUnit.SECONDS)); + Thread.sleep(400L); + assertTrue(invocationCount.get() >= 3); + assertTrue(invocationCount.get() <= 10, "Empty source invocation count was " + invocationCount.get()); + } finally { + unscheduleConnectable(connectable, lifecycleState); + } + } + + @Test + void testAutomaticSchedulingBypassesSourceBackoffForLocalInputAndRemovesListener() throws InterruptedException { + final AtomicInteger invocationCount = new AtomicInteger(); + final CountDownLatch firstInvocation = new CountDownLatch(1); + final Connectable connectable = createMockedConnectable(1, SchedulingStrategy.AUTO, invocationCount, firstInvocation); + final Connection connection = mock(Connection.class); + final FlowFileQueue queue = mock(FlowFileQueue.class); + final QueueSchedulingRegistration registration = mock(QueueSchedulingRegistration.class); + final AtomicInteger listenerRegistrations = new AtomicInteger(); + final CountDownLatch listenerRefreshed = new CountDownLatch(1); + configureIncomingConnection(connectable, connection, queue); + when(queue.getLocalQueueSize()).thenReturn(new QueueSize(1, 1L)); + when(queue.isActiveQueueEmpty()).thenReturn(false); + when(queue.getFlowFileAvailability()).thenReturn(FlowFileAvailability.FLOWFILE_AVAILABLE); + when(queue.addSchedulingListener(any())).thenAnswer(invocation -> { + if (listenerRegistrations.incrementAndGet() == 2) { + listenerRefreshed.countDown(); + } + + return registration; + }); + final LifecycleState lifecycleState = new LifecycleState(COMPONENT_ID); + + scheduleConnectable(connectable, lifecycleState); + try { + assertTrue(firstInvocation.await(2, TimeUnit.SECONDS)); + Thread.sleep(200L); + assertTrue(invocationCount.get() > 10, "Local input invocation count was " + invocationCount.get()); + agent.onEvent(connectable); + assertTrue(listenerRefreshed.await(2, TimeUnit.SECONDS)); + } finally { + unscheduleConnectable(connectable, lifecycleState); + } + + verify(registration, times(2)).close(); + } + + @Test + void testAutomaticSchedulingDoesNotBypassProcessorYield() throws InterruptedException { + final AtomicInteger invocationCount = new AtomicInteger(); + final CountDownLatch firstInvocation = new CountDownLatch(1); + final AtomicLong yieldExpiration = new AtomicLong(); + final Connectable connectable = createMockedConnectable(1, SchedulingStrategy.AUTO, invocationCount, firstInvocation); + when(connectable.getYieldExpiration()).thenAnswer(invocation -> yieldExpiration.get()); + doAnswer(invocation -> { + invocationCount.incrementAndGet(); + yieldExpiration.set(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(1L)); + firstInvocation.countDown(); + return null; + }).when(connectable).onTrigger(any(), any()); + final LifecycleState lifecycleState = new LifecycleState(COMPONENT_ID); + + scheduleConnectable(connectable, lifecycleState); + try { + assertTrue(firstInvocation.await(2, TimeUnit.SECONDS)); + Thread.sleep(300L); + assertEquals(1, invocationCount.get()); + } finally { + unscheduleConnectable(connectable, lifecycleState); + } + } + + @Test + void testAutomaticSchedulingWakesAtPenaltyDeadlineWithoutQueueMutation() throws InterruptedException { + final CountDownLatch invocation = new CountDownLatch(1); + final Connectable connectable = createMockedConnectable(1, SchedulingStrategy.AUTO, new AtomicInteger(), invocation); + final Connection connection = mock(Connection.class); + final FlowFileQueue queue = mock(FlowFileQueue.class); + final Instant penaltyExpiration = Instant.now().plusMillis(300L); + configureIncomingConnection(connectable, connection, queue); + when(queue.addSchedulingListener(any())).thenReturn(QueueSchedulingRegistration.NO_OP); + when(queue.getNextFlowFileAvailabilityTime()).thenReturn(penaltyExpiration); + when(queue.getFlowFileAvailability()).thenAnswer(ignored -> Instant.now().isBefore(penaltyExpiration) + ? FlowFileAvailability.HEAD_OF_QUEUE_PENALIZED : FlowFileAvailability.FLOWFILE_AVAILABLE); + when(connectable.isTriggerWhenEmpty()).thenReturn(false); + when(connectable.hasIncomingConnection()).thenReturn(true); + final LifecycleState lifecycleState = new LifecycleState(COMPONENT_ID); + + scheduleConnectable(connectable, lifecycleState); + try { + assertFalse(invocation.await(150L, TimeUnit.MILLISECONDS)); + assertTrue(invocation.await(2L, TimeUnit.SECONDS)); + } finally { + unscheduleConnectable(connectable, lifecycleState); + } + } + + @Test + void testAutomaticSchedulingGenerationWakesForQueueListenerNotification() throws InterruptedException { + final AtomicBoolean inputAvailable = new AtomicBoolean(); + final AtomicReference listenerReference = new AtomicReference<>(); + final CountDownLatch invocation = new CountDownLatch(1); + final Connectable connectable = createMockedConnectable(1, SchedulingStrategy.AUTO, new AtomicInteger(), invocation); + final Connection connection = mock(Connection.class); + final FlowFileQueue queue = mock(FlowFileQueue.class); + configureIncomingConnection(connectable, connection, queue); + when(connectable.isTriggerWhenEmpty()).thenReturn(false); + when(queue.getLocalQueueSize()).thenAnswer(ignored -> inputAvailable.get() ? new QueueSize(1, 1L) : new QueueSize(0, 0L)); + when(queue.isActiveQueueEmpty()).thenAnswer(ignored -> !inputAvailable.get()); + when(queue.getFlowFileAvailability()).thenAnswer(ignored -> inputAvailable.get() + ? FlowFileAvailability.FLOWFILE_AVAILABLE : FlowFileAvailability.ACTIVE_QUEUE_EMPTY); + when(queue.addSchedulingListener(any())).thenAnswer(invocationOnMock -> { + listenerReference.set(invocationOnMock.getArgument(0, QueueSchedulingListener.class)); + return QueueSchedulingRegistration.NO_OP; + }); + final LifecycleState lifecycleState = new LifecycleState(COMPONENT_ID); + + scheduleConnectable(connectable, lifecycleState); + try { + waitForAutoCount(1, () -> agent.getAutoSchedulingWaiterCount(COMPONENT_ID)); + inputAvailable.set(true); + listenerReference.get().onQueueStateChanged(); + assertTrue(invocation.await(2L, TimeUnit.SECONDS)); + } finally { + unscheduleConnectable(connectable, lifecycleState); + } + } + + @Test + void testAutomaticSchedulingStartsConcurrentWorkBeforeFirstInvocationCompletes() throws InterruptedException { + final CountDownLatch invocationsStarted = new CountDownLatch(2); + final CountDownLatch releaseInvocations = new CountDownLatch(1); + final Connectable connectable = createMockedConnectable(1, SchedulingStrategy.AUTO, new AtomicInteger(), new CountDownLatch(0)); + doAnswer(invocation -> { + invocationsStarted.countDown(); + releaseInvocations.await(); + return null; + }).when(connectable).onTrigger(any(), any()); + final LifecycleState lifecycleState = new LifecycleState(COMPONENT_ID); + + scheduleConnectable(connectable, lifecycleState); + try { + assertTrue(invocationsStarted.await(4, TimeUnit.SECONDS)); + final AutoSchedulingDiagnostics diagnostics = agent.getAutoSchedulingDiagnostics(COMPONENT_ID); + assertEquals(2, diagnostics.activeProcessorInvocations()); + assertFalse(diagnostics.flowFileMeasurementsAvailable()); + } finally { + releaseInvocations.countDown(); + unscheduleConnectable(connectable, lifecycleState); + } + } + + @Test + void testAutomaticWorkersScaleAndRetireAfterLongInvocations() throws InterruptedException { + final CountDownLatch invocationStarted = new CountDownLatch(1); + final CountDownLatch releaseInvocations = new CountDownLatch(1); + final Connectable connectable = createMockedConnectable(1, SchedulingStrategy.AUTO, new AtomicInteger(), new CountDownLatch(0)); + doAnswer(invocation -> { + invocationStarted.countDown(); + releaseInvocations.await(); + return null; + }).when(connectable).onTrigger(any(), any()); + final LifecycleState lifecycleState = new LifecycleState(COMPONENT_ID); + + scheduleConnectable(connectable, lifecycleState); + try { + agent.requestAutoSchedulingSettings(COMPONENT_ID, new SchedulingSettings(3, 0L)); + assertTrue(invocationStarted.await(2L, TimeUnit.SECONDS)); + waitForRunningThreadCount(3, 2L, TimeUnit.SECONDS); + assertEquals(3, agent.getAutoSchedulingDiagnostics(COMPONENT_ID).currentConcurrentTasks()); + + agent.setMaxThreadCount(1); + waitForAutoCount(1, () -> agent.getAutoSchedulingDiagnostics(COMPONENT_ID).currentConcurrentTasks()); + + releaseInvocations.countDown(); + waitForRunningThreadCount(1, 2L, TimeUnit.SECONDS); + } finally { + releaseInvocations.countDown(); + unscheduleConnectable(connectable, lifecycleState); + } + } + + @Test + void testAutomaticWorkerWaitsForConcurrentTaskSlotReleasedByRetiringInvocation() throws InterruptedException { + final CountDownLatch firstWorkerStarted = new CountDownLatch(1); + final CountDownLatch retiringWorkerStarted = new CountDownLatch(1); + final CountDownLatch releaseFirstWorker = new CountDownLatch(1); + final CountDownLatch releaseRetiringWorker = new CountDownLatch(1); + final CountDownLatch invocationAfterConcurrentTaskSlotReleased = new CountDownLatch(1); + final AtomicInteger invocationCount = new AtomicInteger(); + final AtomicReference listenerReference = new AtomicReference<>(); + final Connectable connectable = createMockedConnectable(1, SchedulingStrategy.AUTO, new AtomicInteger(), new CountDownLatch(0)); + final Connection connection = mock(Connection.class); + final FlowFileQueue queue = mock(FlowFileQueue.class); + configureIncomingConnection(connectable, connection, queue); + when(connectable.isTriggerWhenEmpty()).thenReturn(false); + when(connectable.hasIncomingConnection()).thenReturn(true); + when(queue.getLocalQueueSize()).thenReturn(new QueueSize(1, 1L)); + when(queue.isActiveQueueEmpty()).thenReturn(false); + when(queue.getFlowFileAvailability()).thenReturn(FlowFileAvailability.FLOWFILE_AVAILABLE); + when(queue.addSchedulingListener(any())).thenAnswer(invocation -> { + listenerReference.set(invocation.getArgument(0, QueueSchedulingListener.class)); + return QueueSchedulingRegistration.NO_OP; + }); + doAnswer(invocation -> { + final int currentInvocation = invocationCount.incrementAndGet(); + if (currentInvocation > 2) { + invocationAfterConcurrentTaskSlotReleased.countDown(); + return null; + } + + if (Thread.currentThread().getName().endsWith("task 1")) { + retiringWorkerStarted.countDown(); + releaseRetiringWorker.await(); + } else { + firstWorkerStarted.countDown(); + releaseFirstWorker.await(); + } + + return null; + }).when(connectable).onTrigger(any(), any()); + final LifecycleState lifecycleState = new LifecycleState(COMPONENT_ID); + + scheduleConnectable(connectable, lifecycleState); + try { + agent.requestAutoSchedulingSettings(COMPONENT_ID, new SchedulingSettings(2, 0L)); + assertTrue(firstWorkerStarted.await(2L, TimeUnit.SECONDS)); + assertTrue(retiringWorkerStarted.await(2L, TimeUnit.SECONDS)); + + agent.requestAutoSchedulingSettings(COMPONENT_ID, new SchedulingSettings(1, 0L)); + waitForAutoCount(1, () -> agent.getAutoSchedulingDiagnostics(COMPONENT_ID).currentConcurrentTasks()); + releaseFirstWorker.countDown(); + waitForAutoCount(1, () -> agent.getAutoSchedulingConcurrentTaskSlotWaiterCount(COMPONENT_ID)); + assertEquals(2, invocationCount.get()); + assertEquals(MAX_THREADS - 1, agent.getGlobalSemaphore().availablePermits()); + + final long concurrentTaskSlotChangeCount = agent.getAutoSchedulingConcurrentTaskSlotChangeCount(COMPONENT_ID); + for (int signal = 0; signal < 100; signal++) { + listenerReference.get().onQueueStateChanged(); + } + + assertEquals(concurrentTaskSlotChangeCount, agent.getAutoSchedulingConcurrentTaskSlotChangeCount(COMPONENT_ID)); + assertEquals(1, agent.getAutoSchedulingConcurrentTaskSlotWaiterCount(COMPONENT_ID)); + assertEquals(2, invocationCount.get()); + assertEquals(MAX_THREADS - 1, agent.getGlobalSemaphore().availablePermits()); + + releaseRetiringWorker.countDown(); + assertTrue(invocationAfterConcurrentTaskSlotReleased.await(2L, TimeUnit.SECONDS)); + } finally { + releaseFirstWorker.countDown(); + releaseRetiringWorker.countDown(); + unscheduleConnectable(connectable, lifecycleState); + } + } + @Test void testScheduleSpawnsThreadsThatInvoke() throws InterruptedException { final int concurrentTasks = 3; @@ -643,6 +946,16 @@ private void scheduleConnectable(final Connectable connectable, final LifecycleS agent.schedule(connectable, lifecycleState); } + private void configureIncomingConnection(final Connectable connectable, final Connection connection, final FlowFileQueue queue) { + final Connectable source = mock(Connectable.class); + when(source.getIdentifier()).thenReturn("upstream"); + when(connection.getSource()).thenReturn(source); + when(connection.getDestination()).thenReturn(connectable); + when(connection.getFlowFileQueue()).thenReturn(queue); + when(connectable.getIncomingConnections()).thenReturn(List.of(connection)); + when(queue.getNextFlowFileAvailabilityTime()).thenReturn(Instant.EPOCH); + } + private void unscheduleConnectable(final Connectable connectable, final LifecycleState lifecycleState) { lifecycleState.setScheduled(false); agent.unschedule(connectable, lifecycleState); @@ -656,6 +969,15 @@ private void waitForRunningThreadCount(final int expectedCount, final long timeo assertEquals(expectedCount, agent.getRunningThreadCount()); } + private void waitForAutoCount(final int expectedCount, final IntSupplier countSupplier) throws InterruptedException { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2L); + while (countSupplier.getAsInt() != expectedCount && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + + assertEquals(expectedCount, countSupplier.getAsInt()); + } + private Connectable createMockedConnectable(final int maxConcurrentTasks, final SchedulingStrategy schedulingStrategy, final AtomicInteger invocationCount, final CountDownLatch invocationLatch) { final Connectable connectable = mock(Connectable.class); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurementsTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurementsTest.java new file mode 100644 index 000000000000..9a79a0d3a3c8 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurementsTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.CommittedSchedulingWork; +import org.apache.nifi.controller.scheduling.SchedulingSettings; +import org.apache.nifi.controller.tasks.InvocationObserver; +import org.apache.nifi.controller.tasks.InvocationOutcome; +import org.apache.nifi.controller.tasks.InvocationResult; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ProcessorSchedulingMeasurementsTest { + @Test + void testRetainedObserverReportsAllCommitsAcrossConcurrencyChanges() { + final ProcessorSchedulingMeasurements measurements = new ProcessorSchedulingMeasurements(); + final InvocationObserver retainedObserver = measurements.createInvocationObserver(); + retainedObserver.onInvocationCompleted(InvocationResult.completed(InvocationOutcome.INVOKED_WITHOUT_ACTIVITY)); + for (int second = 1; second <= 10; second++) { + retainedObserver.onCommit(new CommittedSchedulingWork(0, 100)); + final ProcessorSchedulingSnapshot snapshot = captureSnapshot(measurements, second); + assertEquals(100, snapshot.committedFlowFiles()); + } + } + + @Test + void testOnlyCommittedWorkContributesToThroughput() { + final ProcessorSchedulingMeasurements measurements = new ProcessorSchedulingMeasurements(); + final InvocationObserver observer = measurements.createInvocationObserver(); + observer.onActivity(); + observer.onCommitFailure(); + observer.onInvocationCompleted(InvocationResult.completed(InvocationOutcome.FAILED)); + final ProcessorSchedulingSnapshot failed = captureSnapshot(measurements, 1); + assertEquals(0, failed.committedFlowFiles()); + assertEquals(1, failed.completedInvocations()); + assertEquals(1, failed.failedInvocations()); + observer.onCommit(new CommittedSchedulingWork(2, 10)); + assertEquals(2, captureSnapshot(measurements, 2).committedFlowFiles()); + assertEquals(0, captureSnapshot(measurements, 3).committedFlowFiles()); + } + + @Test + void testLongInvocationsContributeUtilizationBeforeCompleting() { + final AtomicLong nowNanos = new AtomicLong(); + final ProcessorSchedulingMeasurements measurements = new ProcessorSchedulingMeasurements(nowNanos::get); + final SchedulingSettings settings = new SchedulingSettings(2, 0L); + measurements.recordInvocationStarted(); + + // The scheduling agent reads its timestamp at 1000 milliseconds, then a second invocation starts at 1200 milliseconds before the snapshot is captured. + final long agentTimestampNanos = TimeUnit.MILLISECONDS.toNanos(1000); + nowNanos.set(TimeUnit.MILLISECONDS.toNanos(1200)); + measurements.recordInvocationStarted(); + final ProcessorSchedulingSnapshot first = measurements.captureSnapshot(agentTimestampNanos, agentTimestampNanos, settings, + true, true, 0D, false, false); + // 1200 milliseconds of invocation time out of 1200 milliseconds available to each of 2 tasks + assertEquals(0.5D, first.concurrentTaskUtilization(), 0.000001D); + assertEquals(0, first.completedInvocations()); + + nowNanos.set(TimeUnit.MILLISECONDS.toNanos(2200)); + measurements.recordInvocationFinished(); + nowNanos.set(TimeUnit.MILLISECONDS.toNanos(3200)); + final ProcessorSchedulingSnapshot second = measurements.captureSnapshot(nowNanos.get(), nowNanos.get() - agentTimestampNanos, settings, + true, true, 0D, false, false); + // 2000 milliseconds from the first invocation plus 1000 milliseconds from the second, out of 2000 milliseconds available to each of 2 tasks + assertEquals(0.75D, second.concurrentTaskUtilization(), 0.000001D); + } + + @Test + void testBatchActivityDistinguishesEachTrigger() { + final InvocationObserver observer = new ProcessorSchedulingMeasurements().createInvocationObserver(); + observer.onActivity(); + assertTrue(observer.isTriggerActivityObserved()); + observer.resetTriggerActivity(); + assertFalse(observer.isTriggerActivityObserved()); + assertTrue(observer.isActivityObserved()); + } + + private ProcessorSchedulingSnapshot captureSnapshot(final ProcessorSchedulingMeasurements measurements, final int concurrency) { + return measurements.captureSnapshot(System.nanoTime(), TimeUnit.SECONDS.toNanos(1), + new SchedulingSettings(concurrency, 0L), true, true, 0D, false, false); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingControllerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingControllerTest.java new file mode 100644 index 000000000000..f0d4cdf59dce --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingControllerTest.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.SchedulingSettings; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class StandardAutoSchedulingControllerTest { + private static final SystemSchedulingSnapshot AVAILABLE_SYSTEM_CAPACITY = SystemSchedulingSnapshot.createBuilder() + .setCpuLoadAvailable(true) + .setCpuLoad(0.25D) + .setAverageCpuLoad(0.25D) + .setMaxGlobalConcurrentTasks(32) + .build(); + + @Test + void testQueuedWorkCanAcquireMoreCapacityBeforeFirstCompletion() { + final Simulation simulation = new Simulation(12, false, false); + simulation.completedInvocations = 0; + simulation.observe(0); + assertEquals(2, simulation.settings.concurrentTasks()); + + for (int second = 0; second < 15; second++) { + simulation.observe(0); + assertTrue(simulation.settings.concurrentTasks() <= 2); + } + + simulation.completedInvocations = 1; + simulation.observe(1); + assertEquals(2, simulation.settings.concurrentTasks()); + } + + @ParameterizedTest + @CsvSource({"1000, 1", "2000, 2"}) + void testCatchingUpWithWaitingWorkRequiresThroughputGain(final long comparisonFlowFiles, final int expectedConcurrentTasks) { + final Simulation simulation = new Simulation(12, true, false); + simulation.observe(1000); + assertEquals(2, simulation.settings.concurrentTasks()); + assertEquals(TimeUnit.MILLISECONDS.toNanos(25), simulation.settings.runDurationNanos()); + + simulation.processorReady = false; + simulation.inputQueueHasFlowFiles = false; + simulation.observe(6, comparisonFlowFiles); + assertEquals(expectedConcurrentTasks, simulation.settings.concurrentTasks()); + + simulation.concurrentTaskUtilization = 0.5D; + simulation.observe(1000); + assertEquals(expectedConcurrentTasks, simulation.settings.concurrentTasks()); + } + + @ParameterizedTest + @ValueSource(longs = {0, 100, 1000}) + void testHigherConcurrencyMustImproveUsefulThroughput(final long comparisonFlowFiles) { + final Simulation simulation = new Simulation(12, false, false); + final int originalConcurrentTasks = simulation.settings.concurrentTasks(); + simulation.observe(1000); + assertEquals(2, simulation.settings.concurrentTasks()); + simulation.observe(6, comparisonFlowFiles); + + assertEquals(originalConcurrentTasks, simulation.settings.concurrentTasks()); + assertEquals(ConcurrencyUpdateReason.PREVIOUS_INCREASE_DID_NOT_IMPROVE_THROUGHPUT, + simulation.controller.getConcurrencyUpdateStatus().reason()); + + for (int second = 0; second < 5; second++) { + simulation.observe(1000); + assertEquals(1, simulation.settings.concurrentTasks()); + } + } + + @Test + void testSourceRequiresRecentActivity() { + final Simulation source = new Simulation(12, false, false); + source.inputQueueHasFlowFiles = false; + source.sourceProcessorRecentlyReportedActivity = true; + source.observe(100); + assertEquals(2, source.settings.concurrentTasks()); + + final Simulation idleSource = new Simulation(12, false, false); + idleSource.inputQueueHasFlowFiles = false; + idleSource.concurrentTaskUtilization = 0D; + idleSource.observe(0); + assertEquals(1, idleSource.settings.concurrentTasks()); + } + + @ParameterizedTest + @ValueSource(ints = {2, 16, 64}) + void testConcurrencyAdaptsToConfiguredMaximum(final int maxConcurrentTasks) { + final Simulation simulation = new Simulation(maxConcurrentTasks, false, false); + int acceptedConcurrentTasks = 1; + int maxConcurrentTasksObserved = 1; + for (int second = 0; second < maxConcurrentTasks * 8; second++) { + simulation.observe(simulation.settings.concurrentTasks() * 1000L); + maxConcurrentTasksObserved = Math.max(maxConcurrentTasksObserved, simulation.settings.concurrentTasks()); + if (simulation.lastDecision.reason() == ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_INCREASED_THROUGHPUT) { + assertEquals(acceptedConcurrentTasks + 1, simulation.settings.concurrentTasks()); + acceptedConcurrentTasks = simulation.settings.concurrentTasks(); + } + + assertTrue(simulation.settings.concurrentTasks() <= maxConcurrentTasks); + } + + assertEquals(maxConcurrentTasks, maxConcurrentTasksObserved); + } + + @Test + void testSerialProcessorRemainsSerial() { + final Simulation simulation = new Simulation(12, true, true); + for (int second = 0; second < 10; second++) { + simulation.observe(1000); + assertEquals(1, simulation.settings.concurrentTasks()); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCpuSaturationPreservesProductiveCapacity(final boolean duringHigherConcurrencyTest) { + final Simulation simulation = new Simulation(12, false, false); + simulation.settings = new SchedulingSettings(duringHigherConcurrencyTest ? 1 : 4, 0L); + if (duringHigherConcurrencyTest) { + simulation.observe(1000); + } + + final int expectedConcurrentTasks = duringHigherConcurrencyTest ? 2 : 4; + simulation.systemMeasurements = SystemSchedulingSnapshot.createBuilder() + .setCpuLoadAvailable(true) + .setCpuLoad(0.95D) + .setAverageCpuLoad(0.95D) + .setMaxGlobalConcurrentTasks(32) + .setCpuLoadAboveLimit(true) + .build(); + simulation.observe(15, expectedConcurrentTasks * 1000L); + assertEquals(expectedConcurrentTasks, simulation.settings.concurrentTasks()); + } + + @ParameterizedTest + @CsvSource({"false, 0", "true, 0", "false, 1000"}) + void testIdleBackpressuredAndIntermittentWorkReleaseUnusedCapacity(final boolean backpressured, final long work) { + final Simulation simulation = new Simulation(12, false, false); + simulation.settings = new SchedulingSettings(4, 0L); + simulation.processorReady = !backpressured; + simulation.inputQueueHasFlowFiles = backpressured || work > 0L; + simulation.concurrentTaskUtilization = backpressured ? 1D : 0.4D; + simulation.backpressureNanos = backpressured ? TimeUnit.SECONDS.toNanos(1) : 0L; + for (int second = 0; second < 9; second++) { + final int originalConcurrentTasks = simulation.settings.concurrentTasks(); + simulation.observe(work); + if (simulation.lastDecision.reason() == ProcessorSchedulingDecisionReason.REDUCED_UNUSED_CONCURRENT_TASK) { + assertEquals(originalConcurrentTasks - 1, simulation.settings.concurrentTasks()); + } + } + + assertEquals(1, simulation.settings.concurrentTasks()); + + simulation.processorReady = true; + simulation.inputQueueHasFlowFiles = true; + simulation.concurrentTaskUtilization = 1D; + simulation.backpressureNanos = 0L; + simulation.observe(1000); + assertEquals(2, simulation.settings.concurrentTasks()); + } + + @Test + void testReductionPreservesThroughputAndRestoresCapacityWhenNeeded() { + final Simulation simulation = new Simulation(12, false, false); + simulation.settings = new SchedulingSettings(4, 0L); + simulation.concurrentTaskUtilization = 0.5D; + simulation.observe(30, 1000); + assertEquals(4, simulation.settings.concurrentTasks()); + + simulation.observe(1000); + assertEquals(ProcessorSchedulingDecisionReason.TESTING_LOWER_CONCURRENCY, simulation.lastDecision.reason()); + assertEquals(3, simulation.settings.concurrentTasks()); + simulation.observe(6, 1000); + assertEquals(ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_MAINTAINED_THROUGHPUT, simulation.lastDecision.reason()); + assertEquals(3, simulation.settings.concurrentTasks()); + + // Each lower setting is compared against the rate measured with 4 tasks, not against the previous lower setting. + simulation.observe(3, 960); + assertEquals(ProcessorSchedulingDecisionReason.TESTING_LOWER_CONCURRENCY, simulation.lastDecision.reason()); + simulation.observe(6, 960); + assertEquals(ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_MAINTAINED_THROUGHPUT, simulation.lastDecision.reason()); + assertEquals(2, simulation.settings.concurrentTasks()); + + simulation.observe(3, 915); + assertEquals(ProcessorSchedulingDecisionReason.TESTING_LOWER_CONCURRENCY, simulation.lastDecision.reason()); + simulation.observe(6, 915); + assertEquals(ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_REDUCED_THROUGHPUT_OR_INCREASED_INPUT_QUEUE, simulation.lastDecision.reason()); + assertEquals(2, simulation.settings.concurrentTasks()); + + // A rejected lower setting refreshes the reference rate, so the next test is compared against the current rate. + simulation.observe(30, 915); + assertEquals(ProcessorSchedulingDecisionReason.TESTING_LOWER_CONCURRENCY, simulation.lastDecision.reason()); + simulation.observe(6, 915); + assertEquals(ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_MAINTAINED_THROUGHPUT, simulation.lastDecision.reason()); + assertEquals(1, simulation.settings.concurrentTasks()); + + final Simulation busy = new Simulation(4, false, false); + busy.settings = new SchedulingSettings(4, 0L); + busy.observe(30, 4000); + busy.observe(4000); + assertEquals(ProcessorSchedulingDecisionReason.TESTING_LOWER_CONCURRENCY, busy.lastDecision.reason()); + assertEquals(3, busy.settings.concurrentTasks()); + + busy.observe(6, 3000); + assertEquals(ProcessorSchedulingDecisionReason.LOWER_CONCURRENCY_REDUCED_THROUGHPUT_OR_INCREASED_INPUT_QUEUE, busy.lastDecision.reason()); + assertEquals(4, busy.settings.concurrentTasks()); + } + + @Test + void testUnavailableCpuMeasurementStillAllowsBoundedProbing() { + final Simulation simulation = new Simulation(12, false, false); + simulation.systemMeasurements = SystemSchedulingSnapshot.createBuilder() + .setCpuLoad(-1D) + .setAverageCpuLoad(-1D) + .setMaxGlobalConcurrentTasks(32) + .build(); + simulation.observe(1000); + assertEquals(2, simulation.settings.concurrentTasks()); + } + + @Test + void testSharedBudgetAndFailuresBoundGrowth() { + final Simulation simulation = new Simulation(12, false, false); + simulation.systemMeasurements = SystemSchedulingSnapshot.createBuilder() + .setCpuLoadAvailable(true) + .setCpuLoad(0.25D) + .setAverageCpuLoad(0.25D) + .setMaxGlobalConcurrentTasks(1) + .setFractionOfSamplesWithAllGlobalTaskSlotsUsed(1D) + .setFractionOfGlobalTaskTimeSpentWaiting(0.5D) + .build(); + simulation.observe(1000); + assertEquals(1, simulation.settings.concurrentTasks()); + assertEquals(ConcurrencyUpdateReason.CONCURRENT_TASK_LIMIT_REACHED, simulation.controller.getConcurrencyUpdateStatus().reason()); + + simulation.systemMeasurements = AVAILABLE_SYSTEM_CAPACITY; + simulation.completedInvocations = 100; + simulation.failedInvocations = 1; + simulation.observe(1000); + assertEquals(ProcessorSchedulingDecisionReason.TESTING_HIGHER_CONCURRENCY, simulation.lastDecision.reason()); + assertEquals(2, simulation.settings.concurrentTasks()); + + simulation.completedInvocations = 10; + simulation.observe(1000); + assertEquals(1, simulation.settings.concurrentTasks()); + simulation.observe(1000); + assertEquals(ProcessorSchedulingDecisionReason.PROCESSOR_INVOCATION_FAILED, simulation.lastDecision.reason()); + } + + @Test + void testConfiguredAndProcessorCountLimitReasons() { + final Simulation configuredLimit = new Simulation(12, false, false); + configuredLimit.settings = new SchedulingSettings(12, 0L); + configuredLimit.observe(12_000); + assertEquals(ConcurrencyUpdateReason.CONFIGURED_MAX_CONCURRENT_TASKS_REACHED, + configuredLimit.controller.getConcurrencyUpdateStatus().reason()); + + final Simulation processorCountLimit = new Simulation(8, false, false, true, 2); + processorCountLimit.settings = new SchedulingSettings(8, 0L); + processorCountLimit.observe(8_000); + assertEquals(ConcurrencyUpdateReason.CPU_LIMIT_REACHED, processorCountLimit.controller.getConcurrencyUpdateStatus().reason()); + } + + @Test + void testWaitingWorkAndResourceLimitReasons() { + final Simulation simulation = new Simulation(12, false, false); + simulation.inputQueueHasFlowFiles = false; + simulation.observe(0); + assertEquals(ConcurrencyUpdateReason.NO_WAITING_WORK, simulation.controller.getConcurrencyUpdateStatus().reason()); + + simulation.inputQueueHasFlowFiles = true; + simulation.systemMeasurements = SystemSchedulingSnapshot.createBuilder() + .setCpuLoadAvailable(true) + .setCpuLoad(0.95D) + .setAverageCpuLoad(0.95D) + .setMaxGlobalConcurrentTasks(32) + .setCpuLoadAboveLimit(true) + .build(); + simulation.observe(1000); + assertEquals(ConcurrencyUpdateReason.CPU_LOAD_TOO_HIGH, simulation.controller.getConcurrencyUpdateStatus().reason()); + } + + @Test + void testConcurrencyComparisonHasBoundedLifetimeWithoutCompletions() { + final Simulation simulation = new Simulation(12, false, false); + simulation.completedInvocations = 0; + simulation.observe(0); + simulation.observe(30, 0); + + assertNotEquals(ConcurrencyEvaluationState.MEASURING_CONCURRENCY_CHANGE, simulation.lastDecision.state()); + assertEquals(1, simulation.settings.concurrentTasks()); + } + + @Test + void testComparisonCoversSeveralInvocationsPerTask() { + final Simulation simulation = new Simulation(12, false, false); + simulation.completedInvocations = 1; + simulation.processorInvocationNanos = TimeUnit.SECONDS.toNanos(1); + simulation.observe(100); + assertEquals(ProcessorSchedulingDecisionReason.TESTING_HIGHER_CONCURRENCY, simulation.lastDecision.reason()); + + simulation.completedInvocations = 2; + simulation.processorInvocationNanos = TimeUnit.SECONDS.toNanos(2); + simulation.observe(9, 200); + assertEquals(ProcessorSchedulingDecisionReason.COLLECTING_COMPARISON_MEASUREMENTS, simulation.lastDecision.reason()); + + simulation.observe(200); + assertEquals(ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_INCREASED_THROUGHPUT, simulation.lastDecision.reason()); + assertEquals(2, simulation.settings.concurrentTasks()); + } + + @Test + void testSlowSerializedWorkDoesNotAccumulateUnproductiveConcurrency() { + final Simulation simulation = new Simulation(12, false, false); + int maxConcurrentTasksObserved = 1; + boolean reducedToOne = false; + for (int second = 1; second <= 120; second++) { + final boolean completed = second % 5 == 0; + simulation.completedInvocations = completed ? 1 : 0; + simulation.processorInvocationNanos = completed ? TimeUnit.SECONDS.toNanos(5L * simulation.settings.concurrentTasks()) : 0L; + simulation.observe(completed ? 100 : 0); + maxConcurrentTasksObserved = Math.max(maxConcurrentTasksObserved, simulation.settings.concurrentTasks()); + if (second > 60 && simulation.settings.concurrentTasks() == 1) { + reducedToOne = true; + } + } + + assertTrue(maxConcurrentTasksObserved <= 3); + assertTrue(reducedToOne); + } + + @Test + void testScalableWorkloadRampsDespiteVariableCommitRates() { + final Simulation simulation = new Simulation(16, false, false); + final double[] relativeRates = {1.3D, 0.7D, 1.2D, 0.8D, 1.1D, 0.9D}; + int maxConcurrentTasksObserved = 1; + for (int second = 0; second < 180; second++) { + simulation.observe((long) (1000 * simulation.settings.concurrentTasks() * relativeRates[second % relativeRates.length])); + maxConcurrentTasksObserved = Math.max(maxConcurrentTasksObserved, simulation.settings.concurrentTasks()); + } + + assertEquals(16, maxConcurrentTasksObserved); + } + + private static class Simulation { + private final StandardAutoSchedulingController controller; + private SchedulingSettings settings; + private SystemSchedulingSnapshot systemMeasurements = AVAILABLE_SYSTEM_CAPACITY; + private ProcessorSchedulingDecision lastDecision; + private long timestampNanos; + private long completedInvocations = 10; + private long failedInvocations; + private long processorInvocationNanos; + private long backpressureNanos; + private double concurrentTaskUtilization = 1D; + private boolean processorReady = true; + private boolean inputQueueHasFlowFiles = true; + private boolean sourceProcessorRecentlyReportedActivity; + + private Simulation(final int maxConcurrentTasks, final boolean batchingSupported, final boolean processorTriggeredSerially) { + this(maxConcurrentTasks, batchingSupported, processorTriggeredSerially, false, Runtime.getRuntime().availableProcessors()); + } + + private Simulation(final int maxConcurrentTasks, final boolean batchingSupported, final boolean processorTriggeredSerially, + final boolean maxConcurrentTasksBasedOnAvailableProcessors, final int availableProcessorCount) { + controller = new StandardAutoSchedulingController(maxConcurrentTasks, batchingSupported, processorTriggeredSerially, + maxConcurrentTasksBasedOnAvailableProcessors, availableProcessorCount); + systemMeasurements = SystemSchedulingSnapshot.createBuilder() + .setCpuLoadAvailable(true) + .setCpuLoad(0.25D) + .setAverageCpuLoad(0.25D) + .setMaxGlobalConcurrentTasks(Math.max(32, maxConcurrentTasks)) + .build(); + settings = new SchedulingSettings(1, batchingSupported ? TimeUnit.MILLISECONDS.toNanos(25) : 0L); + } + + private void observe(final int seconds, final long work) { + for (int second = 0; second < seconds; second++) { + observe(work); + } + } + + private void observe(final long committedFlowFiles) { + timestampNanos += TimeUnit.SECONDS.toNanos(1); + final ProcessorSchedulingSnapshot processorMeasurements = ProcessorSchedulingSnapshot.createBuilder() + .setTimestampNanos(timestampNanos) + .setMeasurementWindowNanos(TimeUnit.SECONDS.toNanos(1)) + .setAppliedSettings(settings) + .setCommittedFlowFiles(committedFlowFiles) + .setCompletedInvocations(completedInvocations) + .setFailedInvocations(failedInvocations) + .setProcessorInvocationNanos(processorInvocationNanos) + .setBackpressureNanos(backpressureNanos) + .setConcurrentTaskUtilization(concurrentTaskUtilization) + .setProcessorReady(processorReady) + .setInputQueueHasFlowFiles(inputQueueHasFlowFiles) + .setSourceProcessorRecentlyReportedActivity(sourceProcessorRecentlyReportedActivity) + .build(); + lastDecision = controller.evaluate(processorMeasurements, systemMeasurements); + if (lastDecision.applySettings()) { + settings = lastDecision.settings(); + } + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingMetricsTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingMetricsTest.java new file mode 100644 index 000000000000..d54ca1240277 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingMetricsTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import com.sun.management.OperatingSystemMXBean; +import org.junit.jupiter.api.Test; + +import java.lang.management.GarbageCollectorMXBean; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class SystemSchedulingMetricsTest { + private static final int MAX_GLOBAL_CONCURRENT_TASKS = 32; + + @Test + void testCpuLoadLimitUsesAverageAndResumeReadings() { + final OperatingSystemMXBean operatingSystemMXBean = mock(OperatingSystemMXBean.class); + when(operatingSystemMXBean.getProcessCpuLoad()).thenReturn(0.995D, 0.50D, 0.50D); + final AtomicLong nowNanos = new AtomicLong(); + final SystemSchedulingMetrics metrics = new SystemSchedulingMetrics(operatingSystemMXBean, List.of(), nowNanos::get); + + assertTrue(captureSnapshotAfterOneSecond(metrics, nowNanos).cpuLoadAboveLimit()); + assertTrue(captureSnapshotAfterOneSecond(metrics, nowNanos).cpuLoadAboveLimit()); + assertTrue(captureSnapshotAfterOneSecond(metrics, nowNanos).cpuLoadAboveLimit()); + assertFalse(captureSnapshotAfterOneSecond(metrics, nowNanos).cpuLoadAboveLimit()); + } + + @Test + void testGarbageCollectionLimitRequiresTwoHighReadings() { + final OperatingSystemMXBean operatingSystemMXBean = mock(OperatingSystemMXBean.class); + when(operatingSystemMXBean.getProcessCpuLoad()).thenReturn(0.25D); + final GarbageCollectorMXBean garbageCollectorMXBean = mock(GarbageCollectorMXBean.class); + when(garbageCollectorMXBean.getName()).thenReturn("G1 Young Generation"); + when(garbageCollectorMXBean.getCollectionTime()).thenReturn(0L, 100L, 200L); + final AtomicLong nowNanos = new AtomicLong(); + final SystemSchedulingMetrics metrics = new SystemSchedulingMetrics( + operatingSystemMXBean, List.of(garbageCollectorMXBean), nowNanos::get); + + assertFalse(captureSnapshotAfterOneSecond(metrics, nowNanos).garbageCollectionTimeAboveLimit()); + assertTrue(captureSnapshotAfterOneSecond(metrics, nowNanos).garbageCollectionTimeAboveLimit()); + + // Collectors such as ZGC report the duration of collection cycles that run alongside the application; only pauses count. + final GarbageCollectorMXBean cycleCollectorMXBean = mock(GarbageCollectorMXBean.class); + when(cycleCollectorMXBean.getName()).thenReturn("ZGC Major Cycles"); + when(cycleCollectorMXBean.getCollectionTime()).thenReturn(0L, 900L, 1800L); + final SystemSchedulingMetrics cycleMetrics = new SystemSchedulingMetrics( + operatingSystemMXBean, List.of(cycleCollectorMXBean), nowNanos::get); + + assertFalse(captureSnapshotAfterOneSecond(cycleMetrics, nowNanos).garbageCollectionTimeAboveLimit()); + assertFalse(captureSnapshotAfterOneSecond(cycleMetrics, nowNanos).garbageCollectionTimeAboveLimit()); + } + + @Test + void testSystemCpuPressureAlsoLimitsGrowth() { + final OperatingSystemMXBean operatingSystemMXBean = mock(OperatingSystemMXBean.class); + when(operatingSystemMXBean.getCpuLoad()).thenReturn(1D); + when(operatingSystemMXBean.getProcessCpuLoad()).thenReturn(0.25D); + final AtomicLong nowNanos = new AtomicLong(); + final SystemSchedulingMetrics metrics = new SystemSchedulingMetrics(operatingSystemMXBean, List.of(), nowNanos::get); + + final SystemSchedulingSnapshot snapshot = captureSnapshotAfterOneSecond(metrics, nowNanos); + + assertEquals(1D, snapshot.cpuLoad()); + assertTrue(snapshot.cpuLoadAboveLimit()); + } + + @Test + void testUnavailableMeasurementDoesNotRetainCpuLoadLimit() { + final OperatingSystemMXBean operatingSystemMXBean = mock(OperatingSystemMXBean.class); + when(operatingSystemMXBean.getCpuLoad()).thenReturn(1D, -1D); + when(operatingSystemMXBean.getProcessCpuLoad()).thenReturn(-1D); + when(operatingSystemMXBean.getProcessCpuTime()).thenReturn(-1L); + final AtomicLong nowNanos = new AtomicLong(); + final SystemSchedulingMetrics metrics = new SystemSchedulingMetrics(operatingSystemMXBean, List.of(), nowNanos::get); + + assertTrue(captureSnapshotAfterOneSecond(metrics, nowNanos).cpuLoadAboveLimit()); + final SystemSchedulingSnapshot unavailable = captureSnapshotAfterOneSecond(metrics, nowNanos); + assertFalse(unavailable.cpuLoadAvailable()); + assertFalse(unavailable.cpuLoadAboveLimit()); + } + + @Test + void testProcessCpuTimeFallback() { + final OperatingSystemMXBean operatingSystemMXBean = mock(OperatingSystemMXBean.class); + when(operatingSystemMXBean.getCpuLoad()).thenReturn(-1D); + when(operatingSystemMXBean.getProcessCpuLoad()).thenReturn(-1D); + when(operatingSystemMXBean.getProcessCpuTime()).thenReturn(0L, TimeUnit.SECONDS.toNanos(2L)); + when(operatingSystemMXBean.getAvailableProcessors()).thenReturn(4); + final AtomicLong nowNanos = new AtomicLong(); + final SystemSchedulingMetrics metrics = new SystemSchedulingMetrics(operatingSystemMXBean, List.of(), nowNanos::get); + + final SystemSchedulingSnapshot snapshot = captureSnapshotAfterOneSecond(metrics, nowNanos); + + assertTrue(snapshot.cpuLoadAvailable()); + assertEquals(0.50D, snapshot.cpuLoad()); + } + + private SystemSchedulingSnapshot captureSnapshotAfterOneSecond(final SystemSchedulingMetrics metrics, final AtomicLong nowNanos) { + nowNanos.addAndGet(TimeUnit.SECONDS.toNanos(1L)); + return metrics.captureSnapshot(MAX_GLOBAL_CONCURRENT_TASKS, 0.25D, 0D); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/tasks/TestConnectableTask.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/tasks/TestConnectableTask.java index a41da4b0c9ad..2b6bccb0b215 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/tasks/TestConnectableTask.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/tasks/TestConnectableTask.java @@ -29,52 +29,39 @@ import org.apache.nifi.controller.queue.FlowFileQueue; import org.apache.nifi.controller.repository.FlowFileEventRepository; import org.apache.nifi.controller.repository.RepositoryContext; +import org.apache.nifi.controller.repository.StandardProcessSession; import org.apache.nifi.controller.repository.StandardRepositoryContext; import org.apache.nifi.controller.scheduling.LifecycleState; import org.apache.nifi.controller.scheduling.RepositoryContextFactory; import org.apache.nifi.controller.scheduling.SchedulingAgent; +import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingMeasurements; import org.apache.nifi.controller.status.FlowFileAvailability; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.processor.Processor; import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.when; public class TestConnectableTask { - private ConnectableTask createTask(final Connectable connectable) { - final FlowController flowController = mock(FlowController.class); - when(flowController.getStateManagerProvider()).thenReturn(mock(StateManagerProvider.class)); - when(flowController.getExtensionManager()).thenReturn(mock(ExtensionManager.class)); - - final RepositoryContext repoContext = mock(StandardRepositoryContext.class); - when(repoContext.getFlowFileEventRepository()).thenReturn(mock(FlowFileEventRepository.class)); - - when(flowController.getGarbageCollectionLog()).thenReturn(mock(GarbageCollectionLog.class)); - - final RepositoryContextFactory contextFactory = mock(RepositoryContextFactory.class); - when(contextFactory.newProcessContext(any(Connectable.class), any(AtomicLong.class))).thenReturn(repoContext); - - final LifecycleState lifecycleState = new LifecycleState(connectable.getIdentifier()); - lifecycleState.setScheduled(true); - - return new ConnectableTask(mock(SchedulingAgent.class), connectable, - flowController, contextFactory, lifecycleState); - } - @Test public void testInvokeDoesNotHoldTaskMonitorDuringProcessorInvocation() { final ProcessorNode processorNode = mock(ProcessorNode.class); @@ -102,6 +89,31 @@ public void testInvokeDoesNotHoldTaskMonitorDuringProcessorInvocation() { assertFalse(taskMonitorHeld.get()); } + @Test + public void testAutomaticBatchEndsAtFirstEmptyTrigger() { + final ProcessorNode processorNode = mock(ProcessorNode.class); + when(processorNode.getIdentifier()).thenReturn("batch-source"); + when(processorNode.getRunnableComponent()).thenReturn(mock(Processor.class)); + when(processorNode.getRelationships()).thenReturn(Collections.emptySet()); + when(processorNode.getIncomingConnections()).thenReturn(Collections.emptyList()); + when(processorNode.getScheduledState()).thenReturn(ScheduledState.RUNNING); + when(processorNode.isSessionBatchingSupported()).thenReturn(true); + final InvocationObserver observer = new ProcessorSchedulingMeasurements().createInvocationObserver(); + final AtomicInteger invocations = new AtomicInteger(); + doAnswer(invocation -> { + if (invocations.incrementAndGet() == 1) { + observer.onActivity(); + } + + return null; + }).when(processorNode).onTrigger(any(), any()); + + try (final MockedConstruction ignored = mockConstruction(StandardProcessSession.class)) { + createTask(processorNode).invoke(TimeUnit.SECONDS.toNanos(1), () -> true, observer); + assertEquals(2, invocations.get()); + } + } + @Test public void testIsWorkToDo() { final ProcessorNode procNode = mock(ProcessorNode.class); @@ -217,4 +229,24 @@ public void testIsWorkToDoFunnels() { "When a Funnel has both incoming and outgoing connections and FlowFiles to process," + " then it should be executed."); } + + private ConnectableTask createTask(final Connectable connectable) { + final FlowController flowController = mock(FlowController.class); + when(flowController.getStateManagerProvider()).thenReturn(mock(StateManagerProvider.class)); + when(flowController.getExtensionManager()).thenReturn(mock(ExtensionManager.class)); + + final RepositoryContext repoContext = mock(StandardRepositoryContext.class); + when(repoContext.getFlowFileEventRepository()).thenReturn(mock(FlowFileEventRepository.class)); + + when(flowController.getGarbageCollectionLog()).thenReturn(mock(GarbageCollectionLog.class)); + + final RepositoryContextFactory contextFactory = mock(RepositoryContextFactory.class); + when(contextFactory.newProcessContext(any(Connectable.class), any(AtomicLong.class))).thenReturn(repoContext); + + final LifecycleState lifecycleState = new LifecycleState(connectable.getIdentifier()); + lifecycleState.setScheduled(true); + + return new ConnectableTask(mock(SchedulingAgent.class), connectable, + flowController, contextFactory, lifecycleState); + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index 390c6585d43a..13cfb0e39b72 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -36,6 +36,7 @@ 30 sec 10 millis AUTO + 12 10000 1 GB diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index bbe41a671358..a468423ad377 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -28,6 +28,8 @@ nifi.administrative.yield.duration=${nifi.administrative.yield.duration} # If a component has no work to do (is "bored"), how long should we wait before checking again for work? nifi.bored.yield.duration=${nifi.bored.yield.duration} nifi.scheduling.strategy=${nifi.scheduling.strategy} +# Per-Processor AUTO concurrency is capped at the smaller of this value and four times the available processor count. +nifi.processor.auto.max.concurrent.tasks=${nifi.processor.auto.max.concurrent.tasks} nifi.queue.backpressure.count=${nifi.queue.backpressure.count} nifi.queue.backpressure.size=${nifi.queue.backpressure.size} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index ad76675bf187..e818f506de0e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -3087,6 +3087,7 @@ public boolean isRemoteGroupPortConnected(final String remoteProcessGroupId, fin @Override public void verifyComponentTypes(final VersionedProcessGroup versionedGroup) { + controllerFacade.verifyComponentTypes(versionedGroup); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 967abbd5a7a0..666f318b3090 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -120,6 +120,7 @@ import org.apache.nifi.controller.repository.FlowFileRecord; import org.apache.nifi.controller.repository.claim.ContentClaim; import org.apache.nifi.controller.repository.claim.ResourceClaim; +import org.apache.nifi.controller.scheduling.auto.AutoSchedulingDiagnostics; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.controller.state.SortedStateUtils; @@ -212,6 +213,7 @@ import org.apache.nifi.web.api.dto.action.details.ConnectDetailsDTO; import org.apache.nifi.web.api.dto.action.details.MoveDetailsDTO; import org.apache.nifi.web.api.dto.action.details.PurgeDetailsDTO; +import org.apache.nifi.web.api.dto.diagnostics.AutoSchedulingDiagnosticsDTO; import org.apache.nifi.web.api.dto.diagnostics.ClassLoaderDiagnosticsDTO; import org.apache.nifi.web.api.dto.diagnostics.ConnectionDiagnosticsDTO; import org.apache.nifi.web.api.dto.diagnostics.ConnectionDiagnosticsSnapshotDTO; @@ -3613,6 +3615,7 @@ private ProcessorDTO createProcessorDto(final ProcessorNode node, final boolean dto.setDescription(getCapabilityDescription(node.getClass())); dto.setSupportsParallelProcessing(!node.isTriggeredSerially()); dto.setSupportsBatching(node.isSessionBatchingSupported()); + dto.setSupportsAutoScheduling(node.isAutoSchedulingSupported()); dto.setConfig(createProcessorConfigDto(node, uiOnly)); @@ -4064,6 +4067,24 @@ public ProcessorDiagnosticsDTO createProcessorDiagnosticsDto(final ProcessorNode procDiagnostics.setProcessor(createProcessorDto(procNode)); procDiagnostics.setProcessorStatus(createProcessorStatusDto(procStatus)); procDiagnostics.setThreadDumps(createThreadDumpDtos(procNode)); + final AutoSchedulingDiagnostics autoSchedulingDiagnostics = flowController.getAutoSchedulingDiagnostics(procNode); + if (autoSchedulingDiagnostics != null) { + final AutoSchedulingDiagnosticsDTO autoSchedulingDiagnosticsDto = new AutoSchedulingDiagnosticsDTO(); + autoSchedulingDiagnosticsDto.setExecutionMode(autoSchedulingDiagnostics.executionMode()); + autoSchedulingDiagnosticsDto.setMaxConcurrentTasks(autoSchedulingDiagnostics.maxConcurrentTasks()); + autoSchedulingDiagnosticsDto.setCurrentConcurrentTasks(autoSchedulingDiagnostics.currentConcurrentTasks()); + autoSchedulingDiagnosticsDto.setActiveProcessorInvocations(autoSchedulingDiagnostics.activeProcessorInvocations()); + autoSchedulingDiagnosticsDto.setCurrentRunDurationMillis(autoSchedulingDiagnostics.currentRunDurationMillis()); + autoSchedulingDiagnosticsDto.setConcurrencyEvaluationState(autoSchedulingDiagnostics.concurrencyEvaluationState()); + autoSchedulingDiagnosticsDto.setConcurrencyUpdateReason(autoSchedulingDiagnostics.concurrencyUpdateReason()); + autoSchedulingDiagnosticsDto.setFlowFilesPerSecond(autoSchedulingDiagnostics.flowFilesPerSecond()); + autoSchedulingDiagnosticsDto.setMeasurementWindowMillis(autoSchedulingDiagnostics.measurementWindowMillis()); + autoSchedulingDiagnosticsDto.setLastConcurrencyUpdateReason(autoSchedulingDiagnostics.lastConcurrencyUpdateReason()); + autoSchedulingDiagnosticsDto.setCollectingMeasurements(autoSchedulingDiagnostics.collectingMeasurements()); + autoSchedulingDiagnosticsDto.setFlowFileMeasurementsAvailable(autoSchedulingDiagnostics.flowFileMeasurementsAvailable()); + autoSchedulingDiagnosticsDto.setConcurrencyIncreaseExplanation(autoSchedulingDiagnostics.concurrencyIncreaseExplanation()); + procDiagnostics.setAutoSchedulingDiagnostics(autoSchedulingDiagnosticsDto); + } final Set referencedServiceDiagnostics = createReferencedServiceDiagnostics(procNode.getEffectivePropertyValues(), flowController.getControllerServiceProvider(), serviceEntityFactory); @@ -4449,11 +4470,17 @@ public ProcessorConfigDTO createProcessorConfigDto(final ProcessorNode procNode, final Map defaultConcurrentTasks = new HashMap<>(); defaultConcurrentTasks.put(SchedulingStrategy.TIMER_DRIVEN.name(), String.valueOf(SchedulingStrategy.TIMER_DRIVEN.getDefaultConcurrentTasks())); defaultConcurrentTasks.put(SchedulingStrategy.CRON_DRIVEN.name(), String.valueOf(SchedulingStrategy.CRON_DRIVEN.getDefaultConcurrentTasks())); + if (procNode.isAutoSchedulingSupported()) { + defaultConcurrentTasks.put(SchedulingStrategy.AUTO.name(), String.valueOf(SchedulingStrategy.AUTO.getDefaultConcurrentTasks())); + } dto.setDefaultConcurrentTasks(defaultConcurrentTasks); final Map defaultSchedulingPeriod = new HashMap<>(); defaultSchedulingPeriod.put(SchedulingStrategy.TIMER_DRIVEN.name(), SchedulingStrategy.TIMER_DRIVEN.getDefaultSchedulingPeriod()); defaultSchedulingPeriod.put(SchedulingStrategy.CRON_DRIVEN.name(), SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod()); + if (procNode.isAutoSchedulingSupported()) { + defaultSchedulingPeriod.put(SchedulingStrategy.AUTO.name(), SchedulingStrategy.AUTO.getDefaultSchedulingPeriod()); + } dto.setDefaultSchedulingPeriod(defaultSchedulingPeriod); } @@ -4678,6 +4705,7 @@ public ProcessorDTO copy(final ProcessorDTO original) { copy.setBundle(copy(original.getBundle())); copy.setSupportsParallelProcessing(original.getSupportsParallelProcessing()); copy.setSupportsBatching(original.getSupportsBatching()); + copy.setSupportsAutoScheduling(original.getSupportsAutoScheduling()); copy.setSupportsSensitiveDynamicProperties(original.getSupportsSensitiveDynamicProperties()); copy.setSupportsBacklogReporting(original.getSupportsBacklogReporting()); copy.setPersistsState(original.getPersistsState()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java index 770115e864db..c6278753fdcd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java @@ -157,8 +157,8 @@ public ProcessorNode createProcessor(final String groupId, ProcessorDTO processo // Notify the processor node that the configuration (properties, e.g.) has been restored final Class componentClass = processor.getProcessor() == null ? null : processor.getProcessor().getClass(); - final StandardProcessContext processContext = new StandardProcessContext(processor, flowController.getControllerServiceProvider(), - flowController.getStateManagerProvider().getStateManager(processor.getProcessor().getIdentifier(), componentClass), () -> false, flowController); + final StandardProcessContext processContext = StandardProcessContext.createBuilder(processor, flowController.getControllerServiceProvider(), + flowController.getStateManagerProvider().getStateManager(processor.getProcessor().getIdentifier(), componentClass), () -> false, flowController).build(); processor.onConfigurationRestored(processContext); return processor; @@ -209,6 +209,7 @@ private void configureProcessor(final ProcessorNode processor, final ProcessorDT if (isNotNull(maxTasks)) { processor.setMaxConcurrentTasks(maxTasks); } + if (isNotNull(schedulingPeriod)) { processor.setSchedulingPeriod(schedulingPeriod); } @@ -319,6 +320,10 @@ private List validateProposedConfiguration(final ProcessorNode processor } } + if (schedulingStrategy == SchedulingStrategy.AUTO && !processorNode.isAutoSchedulingSupported()) { + validationErrors.add("Scheduling strategy AUTO is not supported by Processor " + processorNode.getName() + " [" + processorNode.getIdentifier() + "]"); + } + // validate the concurrent tasks based on the scheduling strategy if (isNotNull(config.getConcurrentlySchedulableTaskCount())) { if (schedulingStrategy == SchedulingStrategy.TIMER_DRIVEN) { @@ -347,6 +352,8 @@ private List validateProposedConfiguration(final ProcessorNode processor throw new IllegalArgumentException(String.format("Scheduling Period '%s' is not a valid cron expression: %s", schedulingPeriod, e.getMessage())); } break; + case AUTO: + break; } } @@ -419,8 +426,8 @@ public Optional getBacklog(final String processorId) throws BacklogRepo final ProcessorNode processor = locateProcessor(processorId); final Processor componentProcessor = processor.getProcessor(); final Class componentClass = componentProcessor == null ? null : componentProcessor.getClass(); - final ProcessContext processContext = new StandardProcessContext(processor, flowController.getControllerServiceProvider(), - flowController.getStateManagerProvider().getStateManager(processor.getIdentifier(), componentClass), () -> false, flowController); + final ProcessContext processContext = StandardProcessContext.createBuilder(processor, flowController.getControllerServiceProvider(), + flowController.getStateManagerProvider().getStateManager(processor.getIdentifier(), componentClass), () -> false, flowController).build(); return processor.getReportedBacklog(processContext); } @@ -528,9 +535,10 @@ private void verifyUpdate(ProcessorNode processor, ProcessorDTO processorDTO) { public List verifyProcessorConfiguration(final String processorId, final Map properties, final Map attributes) { final ProcessorNode processor = locateProcessor(processorId); - final ProcessContext processContext = new StandardProcessContext(processor, properties, processor.getAnnotationData(), - processor.getProcessGroup().getParameterContext(), flowController.getControllerServiceProvider(), - new NopStateManager(), () -> false, flowController); + final ProcessContext processContext = StandardProcessContext.createBuilder( + processor, flowController.getControllerServiceProvider(), new NopStateManager(), () -> false, flowController) + .setPropertyOverrides(properties, processor.getProcessGroup().getParameterContext()) + .build(); final LogRepository logRepository = new NopLogRepository(); final ComponentLog configVerificationLog = new StandardComponentLog( @@ -564,6 +572,7 @@ public ProcessorNode updateProcessor(ProcessorDTO processorDTO) { // configure the processor configureProcessor(processor, processorDTO); parentGroup.onComponentModified(); + flowController.getProcessScheduler().notifySchedulingEvent(processor); // attempt to change the underlying processor if an updated bundle is specified // updating the bundle must happen after configuring so that any additional classpath resources are set first diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardReportingTaskDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardReportingTaskDAO.java index dd87a204d3ba..622ba8c3e8e9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardReportingTaskDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardReportingTaskDAO.java @@ -16,7 +16,6 @@ */ package org.apache.nifi.web.dao.impl; -import org.apache.commons.lang3.StringUtils; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.ConfigurableComponent; @@ -215,10 +214,14 @@ private List validateProposedConfiguration(final ReportingTaskNode repor // this will be the new scheduling strategy so use it schedulingStrategy = SchedulingStrategy.valueOf(reportingTaskDTO.getSchedulingStrategy()); } catch (IllegalArgumentException iae) { - validationErrors.add(String.format("Scheduling strategy: Value must be one of [%s]", StringUtils.join(SchedulingStrategy.values(), ", "))); + validationErrors.add("Scheduling strategy: Value must be one of [TIMER_DRIVEN, CRON_DRIVEN]"); } } + if (schedulingStrategy == SchedulingStrategy.AUTO) { + validationErrors.add("Scheduling strategy AUTO is not supported by Reporting Tasks"); + } + // validate the scheduling period based on the scheduling strategy if (isNotNull(reportingTaskDTO.getSchedulingPeriod())) { switch (schedulingStrategy) { @@ -235,6 +238,8 @@ private List validateProposedConfiguration(final ReportingTaskNode repor throw new IllegalArgumentException(String.format("Scheduling Period '%s' is not a valid cron expression: %s", reportingTaskDTO.getSchedulingPeriod(), e.getMessage())); } break; + case AUTO: + break; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/audit/TestProcessorAuditor.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/audit/TestProcessorAuditor.java index 0603361eca1d..ed539a69b175 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/audit/TestProcessorAuditor.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/audit/TestProcessorAuditor.java @@ -31,6 +31,7 @@ import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.scheduling.StandardProcessScheduler; import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; @@ -101,8 +102,10 @@ class TestProcessorAuditor { private Processor processor; @Mock private ExtensionManager extensionManager; - @Mock + @Mock(strictness = Mock.Strictness.LENIENT) private FlowController flowController; + @Mock(strictness = Mock.Strictness.LENIENT) + private StandardProcessScheduler processScheduler; @Autowired private FlowManager flowManager; @Mock @@ -127,6 +130,7 @@ void init() { when(authentication.getCredentials()).thenReturn(Object.class.getSimpleName()); when(flowController.getFlowManager()).thenReturn(flowManager); + when(flowController.getProcessScheduler()).thenReturn(processScheduler); processorDao.setFlowController(flowController); processorAuditor.setAuditService(auditService); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java index 47f729e17630..87c39730b27b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java @@ -405,6 +405,17 @@ public Resource getResource() { } + @Test + public void testVerifyComponentTypesDelegatesToControllerFacade() { + final ControllerFacade controllerFacade = mock(ControllerFacade.class); + final VersionedProcessGroup versionedGroup = new VersionedProcessGroup(); + serviceFacade.setControllerFacade(controllerFacade); + + serviceFacade.verifyComponentTypes(versionedGroup); + + verify(controllerFacade).verifyComponentTypes(versionedGroup); + } + @Test public void testGetComponentsAffectedByFlowUpdate_WithNewStatelessProcessGroup_ReproducesNPE() { final String groupId = UUID.randomUUID().toString(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java index 0ed1c1632dc1..d0317e31c904 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java @@ -25,13 +25,16 @@ import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ScheduledState; +import org.apache.nifi.controller.exception.ValidationException; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.Processor; +import org.apache.nifi.scheduling.SchedulingStrategy; import org.apache.nifi.web.ResourceNotFoundException; +import org.apache.nifi.web.api.dto.ProcessorConfigDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.dao.ComponentStateDAO; import org.junit.jupiter.api.BeforeEach; @@ -207,6 +210,22 @@ void testVerifyUpdate_ProcessorNotFound() { assertThrows(ResourceNotFoundException.class, () -> dao.verifyUpdate(processorDTO)); } + @Test + void testVerifyUpdateRejectsUnsupportedAutomaticScheduling() { + final ProcessorConfigDTO config = new ProcessorConfigDTO(); + config.setSchedulingStrategy(SchedulingStrategy.AUTO.name()); + final ProcessorDTO processorDTO = new ProcessorDTO(); + processorDTO.setId("test-processor-id"); + processorDTO.setConfig(config); + when(processorNode.isAutoSchedulingSupported()).thenReturn(false); + when(processorNode.getName()).thenReturn("Unsupported Processor"); + + final ValidationException exception = assertThrows(ValidationException.class, () -> dao.verifyUpdate(processorDTO)); + + assertEquals(List.of("Scheduling strategy AUTO is not supported by Processor Unsupported Processor [test-processor-id]"), + exception.getValidationErrors()); + } + @Test void testGetBacklogProvidesRealStateManagerToProcessor() throws Exception { final String processorId = "test-processor-id"; diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.html b/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.html index 3439493853bc..38d68b4b49d3 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.html +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.html @@ -162,46 +162,48 @@

-
-
- - - Concurrent Tasks - - - - -
-
- - - Run Schedule - - - - + @if (!isAutomaticScheduling()) { +
+
+ + + Concurrent Tasks + + + + +
+
+ + + Run Schedule + + + + +
-
+ }
@@ -229,7 +231,7 @@

- @if (supportsBatching()) { + @if (supportsBatching() && !isAutomaticScheduling()) {
Run Duration diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.spec.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.spec.ts index 1a68a2546d19..6f57de7bc156 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.spec.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.spec.ts @@ -831,3 +831,69 @@ describe('EditProcessor with TriggerSerially', () => { expect(concurrentTasks?.disabled).toBe(true); }); }); + +describe('EditProcessor with automatic scheduling', () => { + let component: EditProcessor; + let fixture: ComponentFixture; + + const automaticData: EditComponentDialogRequest = JSON.parse(JSON.stringify(data)); + (automaticData.entity as any).component.supportsAutoScheduling = true; + (automaticData.entity as any).component.config.schedulingStrategy = 'AUTO'; + + beforeEach(() => { + TestBed.configureTestingModule({ + imports: [EditProcessor, MockComponent(ContextErrorBanner), NoopAnimationsModule], + providers: [ + { provide: MAT_DIALOG_DATA, useValue: automaticData }, + { + provide: ClusterConnectionService, + useValue: { + isDisconnectionAcknowledged: vi.fn() + } + }, + { provide: MatDialogRef, useValue: null }, + provideMockStore({ + initialState: { + [errorFeatureKey]: initialErrorState + } + }) + ] + }); + fixture = TestBed.createComponent(EditProcessor); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should offer automatic scheduling and disable manual controls', () => { + expect(component.schedulingStrategies.some((option) => option.value === 'AUTO')).toBe(true); + expect(component.editProcessorForm.get('concurrentTasks')?.disabled).toBe(true); + expect(component.editProcessorForm.get('schedulingPeriod')?.disabled).toBe(true); + }); + + it('should omit manual scheduling values from the update', () => { + vi.spyOn(component.editProcessor, 'next'); + component.submitForm(); + + const update = vi.mocked(component.editProcessor.next).mock.calls[0][0] as any; + expect(update.payload.component.config.schedulingStrategy).toBe('AUTO'); + expect(update.payload.component.config.concurrentlySchedulableTaskCount).toBeUndefined(); + expect(update.payload.component.config.schedulingPeriod).toBeUndefined(); + expect(update.payload.component.config.runDurationMillis).toBeUndefined(); + }); + + it('should restore manual defaults when switching to timer driven', () => { + component.schedulingStrategyChanged('TIMER_DRIVEN'); + + expect(component.editProcessorForm.get('concurrentTasks')?.value).toBe('1'); + expect(component.editProcessorForm.get('schedulingPeriod')?.value).toBe('0 sec'); + expect(component.editProcessorForm.get('concurrentTasks')?.enabled).toBe(true); + expect(component.editProcessorForm.get('schedulingPeriod')?.enabled).toBe(true); + + component.editProcessorForm.get('concurrentTasks')?.setValue('4'); + component.editProcessorForm.get('schedulingPeriod')?.setValue('2 sec'); + component.schedulingStrategyChanged('AUTO'); + component.schedulingStrategyChanged('TIMER_DRIVEN'); + expect(component.editProcessorForm.get('concurrentTasks')?.value).toBe('4'); + expect(component.editProcessorForm.get('schedulingPeriod')?.value).toBe('2 sec'); + }); +}); diff --git a/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.ts b/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.ts index 5870d6bac92b..5e1d5421a506 100644 --- a/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.ts +++ b/nifi-frontend/src/main/frontend/apps/nifi/src/app/ui/common/component-dialogs/edit-processor/edit-processor.component.ts @@ -216,6 +216,14 @@ export class EditProcessor extends TabbedDialog { constructor() { super(); const request = this.request; + if (this.supportsAutoScheduling()) { + this.schedulingStrategies.push({ + text: 'Automatic (Experimental)', + value: 'AUTO', + description: + 'The framework will adjust concurrency, run duration, and invocation timing based on observed work.' + }); + } const processorProperties: any = request.entity.component.config.properties; const properties: Property[] = Object.entries(processorProperties).map((entry: any) => { @@ -242,12 +250,20 @@ export class EditProcessor extends TabbedDialog { concurrentTasks = this.cronDrivenConcurrentTasks; schedulingPeriod = this.cronDrivenSchedulingPeriod; - } else { + } else if (this.schedulingStrategy === 'TIMER_DRIVEN') { this.cronDrivenConcurrentTasks = defaultConcurrentTasks['CRON_DRIVEN']; this.cronDrivenSchedulingPeriod = defaultSchedulingPeriod['CRON_DRIVEN']; this.timerDrivenConcurrentTasks = request.entity.component.config.concurrentlySchedulableTaskCount; this.timerDrivenSchedulingPeriod = request.entity.component.config.schedulingPeriod; + concurrentTasks = this.timerDrivenConcurrentTasks; + schedulingPeriod = this.timerDrivenSchedulingPeriod; + } else { + this.cronDrivenConcurrentTasks = defaultConcurrentTasks['CRON_DRIVEN']; + this.cronDrivenSchedulingPeriod = defaultSchedulingPeriod['CRON_DRIVEN']; + this.timerDrivenConcurrentTasks = defaultConcurrentTasks['TIMER_DRIVEN']; + this.timerDrivenSchedulingPeriod = defaultSchedulingPeriod['TIMER_DRIVEN']; + concurrentTasks = this.timerDrivenConcurrentTasks; schedulingPeriod = this.timerDrivenSchedulingPeriod; } @@ -282,6 +298,7 @@ export class EditProcessor extends TabbedDialog { if (!this.supportsParallelProcessing()) { this.editProcessorForm.get('concurrentTasks')?.disable(); } + this.updateAutomaticSchedulingControls(); if (this.supportsBatching()) { this.editProcessorForm.addControl( @@ -319,6 +336,7 @@ export class EditProcessor extends TabbedDialog { if (!this.supportsParallelProcessing()) { this.editProcessorForm.get('concurrentTasks')?.disable(); } + this.updateAutomaticSchedulingControls(); } private relationshipConfigurationValidator(): ValidatorFn { @@ -358,6 +376,14 @@ export class EditProcessor extends TabbedDialog { return this.request.entity.component.supportsParallelProcessing === true; } + supportsAutoScheduling(): boolean { + return this.request.entity.component.supportsAutoScheduling === true; + } + + isAutomaticScheduling(): boolean { + return this.schedulingStrategy === 'AUTO'; + } + concurrentTasksTooltip(): string { if (this.supportsParallelProcessing()) { return 'The number of tasks that should be concurrently scheduled for this processor. Must be an integer greater than 0.'; @@ -390,15 +416,45 @@ export class EditProcessor extends TabbedDialog { } schedulingStrategyChanged(value: string): void { + if (this.schedulingStrategy === 'CRON_DRIVEN') { + this.cronDrivenConcurrentTasks = this.editProcessorForm.get('concurrentTasks')?.value; + this.cronDrivenSchedulingPeriod = this.editProcessorForm.get('schedulingPeriod')?.value; + } else if (this.schedulingStrategy === 'TIMER_DRIVEN') { + this.timerDrivenConcurrentTasks = this.editProcessorForm.get('concurrentTasks')?.value; + this.timerDrivenSchedulingPeriod = this.editProcessorForm.get('schedulingPeriod')?.value; + } + this.schedulingStrategy = value; if (value === 'CRON_DRIVEN') { this.editProcessorForm.get('concurrentTasks')?.setValue(this.cronDrivenConcurrentTasks); this.editProcessorForm.get('schedulingPeriod')?.setValue(this.cronDrivenSchedulingPeriod); - } else { + } else if (value === 'TIMER_DRIVEN') { this.editProcessorForm.get('concurrentTasks')?.setValue(this.timerDrivenConcurrentTasks); this.editProcessorForm.get('schedulingPeriod')?.setValue(this.timerDrivenSchedulingPeriod); } + + this.updateAutomaticSchedulingControls(); + } + + private updateAutomaticSchedulingControls(): void { + const concurrentTasks = this.editProcessorForm.get('concurrentTasks'); + const schedulingPeriod = this.editProcessorForm.get('schedulingPeriod'); + const runDuration = this.editProcessorForm.get('runDuration'); + if (this.isAutomaticScheduling()) { + concurrentTasks?.disable(); + schedulingPeriod?.disable(); + runDuration?.disable(); + return; + } + + if (!this.readonly && this.supportsParallelProcessing()) { + concurrentTasks?.enable(); + } + if (!this.readonly) { + schedulingPeriod?.enable(); + runDuration?.enable(); + } } executionStrategyDisabled(option: SelectOption): boolean { @@ -436,14 +492,17 @@ export class EditProcessor extends TabbedDialog { yieldDuration: this.editProcessorForm.get('yieldDuration')?.value, bulletinLevel: this.editProcessorForm.get('bulletinLevel')?.value, schedulingStrategy: this.editProcessorForm.get('schedulingStrategy')?.value, - schedulingPeriod: this.editProcessorForm.get('schedulingPeriod')?.value, executionNode: this.editProcessorForm.get('executionNode')?.value, autoTerminatedRelationships: autoTerminated, retriedRelationships: retried, comments: this.editProcessorForm.get('comments')?.value }; - if (this.supportsParallelProcessing()) { + if (!this.isAutomaticScheduling()) { + config.schedulingPeriod = this.editProcessorForm.get('schedulingPeriod')?.value; + } + + if (!this.isAutomaticScheduling() && this.supportsParallelProcessing()) { config.concurrentlySchedulableTaskCount = this.editProcessorForm.get('concurrentTasks')?.value; } @@ -469,7 +528,7 @@ export class EditProcessor extends TabbedDialog { .map((property) => property.descriptor.name); } - if (this.supportsBatching()) { + if (!this.isAutomaticScheduling() && this.supportsBatching()) { payload.component.config.runDurationMillis = this.editProcessorForm.get('runDuration')?.value; } diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java index bc2a2b242337..77dfe080d55f 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java @@ -195,8 +195,8 @@ public ProcessorNode createProcessor(final String type, final String id, final B try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(extensionManager, procNode.getProcessor().getClass(), procNode.getProcessor().getIdentifier())) { final Class componentClass = procNode.getProcessor() == null ? null : procNode.getProcessor().getClass(); final StateManager stateManager = statelessEngine.getStateManagerProvider().getStateManager(id, componentClass); - final StandardProcessContext processContext = new StandardProcessContext(procNode, statelessEngine.getControllerServiceProvider(), - stateManager, () -> false, new StatelessNodeTypeProvider()); + final StandardProcessContext processContext = StandardProcessContext.createBuilder( + procNode, statelessEngine.getControllerServiceProvider(), stateManager, () -> false, new StatelessNodeTypeProvider()).build(); ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnConfigurationRestored.class, procNode.getProcessor(), processContext); } diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessProcessContextFactory.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessProcessContextFactory.java index bc3f3cee414d..5afd65146784 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessProcessContextFactory.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessProcessContextFactory.java @@ -45,7 +45,7 @@ public ProcessContext createProcessContext(final Connectable connectable) { final StateManager stateManager = stateManagerProvider.getStateManager(connectable.getIdentifier(), componentClass); if (connectable instanceof final ProcessorNode processor) { - return new StandardProcessContext(processor, controllerServiceProvider, stateManager, () -> false, NODE_TYPE_PROVIDER); + return StandardProcessContext.createBuilder(processor, controllerServiceProvider, stateManager, () -> false, NODE_TYPE_PROVIDER).build(); } return new ConnectableProcessContext(connectable, stateManager); diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessReloadComponent.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessReloadComponent.java index 95f4518f8120..4e2d69f17be7 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessReloadComponent.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessReloadComponent.java @@ -89,8 +89,8 @@ public void reload(final ProcessorNode existingNode, final String newType, final try (final NarCloseable ignored = NarCloseable.withComponentNarLoader(existingInstanceClassLoader)) { final Class componentClass = existingNode.getProcessor() == null ? null : existingNode.getProcessor().getClass(); final StateManager stateManager = statelessEngine.getStateManagerProvider().getStateManager(id, componentClass); - final StandardProcessContext processContext = new StandardProcessContext(existingNode, statelessEngine.getControllerServiceProvider(), - stateManager, () -> false, new StatelessNodeTypeProvider()); + final StandardProcessContext processContext = StandardProcessContext.createBuilder( + existingNode, statelessEngine.getControllerServiceProvider(), stateManager, () -> false, new StatelessNodeTypeProvider()).build(); ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnRemoved.class, existingNode.getProcessor(), processContext); } finally { @@ -111,8 +111,8 @@ public void reload(final ProcessorNode existingNode, final String newType, final // Notify the processor node that the configuration (properties, e.g.) has been restored final Class componentClass = existingNode.getProcessor() == null ? null : existingNode.getProcessor().getClass(); - final StandardProcessContext processContext = new StandardProcessContext(existingNode, statelessEngine.getControllerServiceProvider(), - statelessEngine.getStateManagerProvider().getStateManager(id, componentClass), () -> false, new StatelessNodeTypeProvider()); + final StandardProcessContext processContext = StandardProcessContext.createBuilder(existingNode, statelessEngine.getControllerServiceProvider(), + statelessEngine.getStateManagerProvider().getStateManager(id, componentClass), () -> false, new StatelessNodeTypeProvider()).build(); existingNode.onConfigurationRestored(processContext); logger.debug("Successfully reloaded {}", existingNode); diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessFlow.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessFlow.java index 8671b8680506..31bb684ec1a8 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessFlow.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessFlow.java @@ -47,6 +47,7 @@ import org.apache.nifi.controller.repository.metrics.NopPerformanceTracker; import org.apache.nifi.controller.repository.metrics.tracking.StatsTracker; import org.apache.nifi.controller.scheduling.LifecycleStateManager; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.controller.service.ControllerServiceState; @@ -810,7 +811,7 @@ public QueueSize enqueue(final InputStream flowFileContents, final Map false, new NopPerformanceTracker()); + final ProcessSessionFactory sessionFactory = new StandardProcessSessionFactory(repositoryContext, () -> false, new NopPerformanceTracker(), SessionSchedulingObserver.NO_OP); final ProcessSession session = sessionFactory.createSession(); try { // Get one of the outgoing connections for the Input Port so that we can return QueueSize for it. diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/session/StatelessProcessSession.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/session/StatelessProcessSession.java index 0434bdebda3e..09246dcefea8 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/session/StatelessProcessSession.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/session/StatelessProcessSession.java @@ -27,6 +27,7 @@ import org.apache.nifi.controller.repository.StandardProcessSession; import org.apache.nifi.controller.repository.metrics.PerformanceTracker; import org.apache.nifi.controller.repository.metrics.ProcessSessionEventBuilder; +import org.apache.nifi.controller.scheduling.SessionSchedulingObserver; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSessionFactory; @@ -64,7 +65,7 @@ public StatelessProcessSession(final Connectable connectable, final RepositoryCo final ExecutionProgress progress, final boolean requireSynchronousCommits, final AsynchronousCommitTracker commitTracker, final PerformanceTracker performanceTracker) { - super(repositoryContextFactory.createRepositoryContext(connectable, provenanceEventRepository), progress::isCanceled, performanceTracker); + super(repositoryContextFactory.createRepositoryContext(connectable, provenanceEventRepository), progress::isCanceled, performanceTracker, SessionSchedulingObserver.NO_OP); this.connectable = connectable; this.repositoryContextFactory = repositoryContextFactory; this.provenanceEventRepository = provenanceEventRepository; diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingIT.java new file mode 100644 index 000000000000..84edfaa1291a --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingIT.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.scheduling; + +import org.apache.nifi.tests.system.NiFiSystemIT; +import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.dto.ProcessorConfigDTO; +import org.apache.nifi.web.api.entity.ConnectionEntity; +import org.apache.nifi.web.api.entity.ProcessorEntity; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AutoSchedulingIT extends NiFiSystemIT { + private static final int BACKPRESSURE_COUNT = 100; + + @Override + protected Map getNifiPropertiesOverrides() { + return Map.of("nifi.scheduling.strategy", "VIRTUAL"); + } + + @Test + @Timeout(value = 1, unit = TimeUnit.MINUTES) + public void testSourceAndQueueConsumerWithAutomaticScheduling() throws NiFiClientException, IOException, InterruptedException { + final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile"); + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); + final ConnectionEntity connection = getClientUtil().createConnection(generate, terminate, "success"); + getClientUtil().updateConnectionBackpressure(connection, BACKPRESSURE_COUNT, 10_000_000L); + + final ProcessorEntity automaticGenerate = getClientUtil().updateProcessorSchedulingStrategy(generate, "AUTO"); + final ProcessorEntity automaticTerminate = getClientUtil().updateProcessorSchedulingStrategy(terminate, "AUTO"); + + assertCanonicalAutoConfiguration(automaticGenerate); + assertCanonicalAutoConfiguration(automaticTerminate); + + getClientUtil().startProcessor(automaticGenerate); + try { + waitForQueueCount(connection, BACKPRESSURE_COUNT); + } finally { + getClientUtil().stopProcessor(automaticGenerate); + } + + getClientUtil().startProcessor(automaticTerminate); + try { + waitForQueueCount(connection, 0); + } finally { + getClientUtil().stopProcessor(automaticTerminate); + } + + assertEquals(0, getConnectionQueueSize(connection.getId())); + } + + @Test + @Timeout(value = 2, unit = TimeUnit.MINUTES) + public void testContinuousFlowAdaptsToBlockingWork() throws NiFiClientException, IOException, InterruptedException { + final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile"); + final ProcessorEntity sleep = getClientUtil().createProcessor("Sleep"); + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); + final ConnectionEntity input = getClientUtil().createConnection(generate, sleep, "success"); + final ConnectionEntity output = getClientUtil().createConnection(sleep, terminate, "success"); + getClientUtil().updateConnectionBackpressure(input, BACKPRESSURE_COUNT, 10_000_000L); + getClientUtil().updateConnectionBackpressure(output, BACKPRESSURE_COUNT, 10_000_000L); + getClientUtil().updateProcessorProperties(sleep, Map.of("onTrigger Sleep Time", "100 ms")); + getClientUtil().updateProcessorSchedulingStrategy(generate, "AUTO"); + getClientUtil().updateProcessorSchedulingStrategy(sleep, "AUTO"); + getClientUtil().updateProcessorSchedulingStrategy(terminate, "AUTO"); + + getClientUtil().startProcessor(terminate); + getClientUtil().startProcessor(sleep); + getClientUtil().startProcessor(generate); + try { + final boolean adaptive = "VIRTUAL".equals(getNifiPropertiesOverrides().get("nifi.scheduling.strategy")); + final int expectedActiveTasks = adaptive ? 4 : 1; + waitFor(() -> getNifiClient().getProcessorClient().getProcessor(sleep.getId()).getStatus().getAggregateSnapshot().getActiveThreadCount() >= expectedActiveTasks); + waitFor(() -> getNifiClient().getProcessorClient().getProcessor(terminate.getId()).getStatus().getAggregateSnapshot().getFlowFilesIn() > 0); + assertCanonicalAutoConfiguration(getNifiClient().getProcessorClient().getProcessor(sleep.getId())); + assertTrue(getConnectionQueueSize(input.getId()) > 0); + + getClientUtil().stopProcessor(generate); + waitForQueueCount(input, 0); + waitForQueueCount(output, 0); + } finally { + getClientUtil().stopProcessor(generate); + getClientUtil().stopProcessor(sleep); + getClientUtil().stopProcessor(terminate); + } + } + + private void assertCanonicalAutoConfiguration(final ProcessorEntity processorEntity) { + final ProcessorConfigDTO config = processorEntity.getComponent().getConfig(); + assertEquals("AUTO", config.getSchedulingStrategy()); + assertEquals(1, config.getConcurrentlySchedulableTaskCount()); + assertEquals("0 sec", config.getSchedulingPeriod()); + assertEquals(0L, config.getRunDurationMillis()); + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingPlatformFallbackIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingPlatformFallbackIT.java new file mode 100644 index 000000000000..d6b33eb8afb1 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingPlatformFallbackIT.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.tests.system.scheduling; + +import java.util.Map; + +public class AutoSchedulingPlatformFallbackIT extends AutoSchedulingIT { + + @Override + protected Map getNifiPropertiesOverrides() { + return Map.of("nifi.scheduling.strategy", "STANDARD"); + } +} From 4d5c521c4a3e18544d4f5f07debe05d4b69dd9a9 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Tue, 29 Sep 2026 13:50:32 -0400 Subject: [PATCH 2/4] NIFI-16365 Move concurrent tasks to Processors with much larger backlogs When the global concurrent task limit is fully used, a Processor with a large backlog could stay at one task while Processors with nearly empty queues held the permits. One concurrent task now moves at a time from such a donor to the receiver: the donor gives up the task before the receiver may test it, and the receiver keeps it only if its throughput increases. Generated with [Snowflake CoCo](https://docs.snowflake.com/en/user-guide/cortex-code/cortex-code) Co-authored-by: Snowflake CoCo --- .../AutoSchedulingDiagnosticsDTO.java | 31 +++ .../VirtualThreadSchedulingAgent.java | 143 ++++++++++--- .../auto/AutoSchedulingDiagnostics.java | 36 ++++ .../auto/AutoSchedulingResetReason.java | 3 +- .../auto/ConcurrentTaskMoveCoordinator.java | 145 +++++++++++++ .../auto/ConcurrentTaskMoveSelector.java | 152 ++++++++++++++ .../auto/ProcessorSchedulingMeasurements.java | 91 ++++---- .../auto/ProcessorSchedulingSnapshot.java | 65 ++++++ .../StandardAutoSchedulingController.java | 18 +- .../auto/SystemSchedulingSnapshot.java | 8 + .../ConcurrentTaskMoveCoordinatorTest.java | 195 ++++++++++++++++++ .../auto/ConcurrentTaskMoveSelectorTest.java | 145 +++++++++++++ .../ProcessorSchedulingMeasurementsTest.java | 37 +++- .../StandardAutoSchedulingControllerTest.java | 39 ++++ .../apache/nifi/web/api/dto/DtoFactory.java | 3 + 15 files changed, 1033 insertions(+), 78 deletions(-) create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveCoordinator.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveSelector.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveCoordinatorTest.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveSelectorTest.java diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/AutoSchedulingDiagnosticsDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/AutoSchedulingDiagnosticsDTO.java index 4835287c12d7..c763848630bf 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/AutoSchedulingDiagnosticsDTO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/diagnostics/AutoSchedulingDiagnosticsDTO.java @@ -34,6 +34,9 @@ public class AutoSchedulingDiagnosticsDTO { private Boolean collectingMeasurements; private Boolean flowFileMeasurementsAvailable; private String concurrencyIncreaseExplanation; + private Long localInputQueueCount; + private Double demandScore; + private String taskMoveRole; @Schema(description = "Automatic scheduling execution mode") public String getExecutionMode() { @@ -140,4 +143,32 @@ public String getConcurrencyIncreaseExplanation() { public void setConcurrencyIncreaseExplanation(final String concurrencyIncreaseExplanation) { this.concurrencyIncreaseExplanation = concurrencyIncreaseExplanation; } + + @Schema(description = "Number of FlowFiles queued on this node across the Processor's incoming connections") + public Long getLocalInputQueueCount() { + return localInputQueueCount; + } + + public void setLocalInputQueueCount(final Long localInputQueueCount) { + this.localInputQueueCount = localInputQueueCount; + } + + @Schema(description = "Score used to move concurrent tasks between Processors: seconds of queued work capped at 60, a fixed 10 for a busy " + + "source Processor, and 0 for a Processor that is failing, stalled, or has nothing queued") + public Double getDemandScore() { + return demandScore; + } + + public void setDemandScore(final Double demandScore) { + this.demandScore = demandScore; + } + + @Schema(description = "RECEIVER or DONOR while this Processor takes part in moving a concurrent task") + public String getTaskMoveRole() { + return taskMoveRole; + } + + public void setTaskMoveRole(final String taskMoveRole) { + this.taskMoveRole = taskMoveRole; + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java index dee3e2c2cbfb..a426d434ea92 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java @@ -28,6 +28,9 @@ import org.apache.nifi.controller.scheduling.auto.AutoSchedulingResetReason; import org.apache.nifi.controller.scheduling.auto.ConcurrencyEvaluationState; import org.apache.nifi.controller.scheduling.auto.ConcurrencyUpdateStatus; +import org.apache.nifi.controller.scheduling.auto.ConcurrentTaskMoveCoordinator; +import org.apache.nifi.controller.scheduling.auto.ConcurrentTaskMoveSelector; +import org.apache.nifi.controller.scheduling.auto.ConcurrentTaskMoveSelector.ProcessorDemand; import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingDecision; import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingDecisionReason; import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingMeasurements; @@ -54,8 +57,10 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -96,6 +101,7 @@ public class VirtualThreadSchedulingAgent implements SchedulingAgent { private final ScheduledExecutorService schedulingEvaluationExecutor; private final SystemSchedulingMetrics systemSchedulingMetrics; private final ConcurrentMap schedulingGenerations = new ConcurrentHashMap<>(); + private final ConcurrentTaskMoveCoordinator concurrentTaskMoveCoordinator = new ConcurrentTaskMoveCoordinator(); private final LongAdder globalPermitHoldNanos = new LongAdder(); private final LongAdder globalPermitWaitNanos = new LongAdder(); private final LongAdder globalTaskSlotsFullyUsedSamples = new LongAdder(); @@ -286,7 +292,9 @@ private void runAutoSchedulingLoop(final ProcessorAutoSchedulingState state, fin } final long concurrentTaskSlotChangeCount = state.generation.getConcurrentTaskSlotChangeCount(); + state.measurements.recordTaskBusy(); if (!acquirePermitWithPolling(state.lifecycleState, state.generation)) { + state.measurements.recordTaskIdle(); return; } @@ -334,6 +342,7 @@ private void runAutoSchedulingLoop(final ProcessorAutoSchedulingState state, fin globalPermitHoldNanos.add(permitHoldNanos); Thread.interrupted(); globalSemaphore.release(); + state.measurements.recordTaskIdle(); } if (concurrentTaskSlotUnavailable) { @@ -498,7 +507,8 @@ private void evaluateAutoScheduling() { final long nowNanos = System.nanoTime(); final int maxGlobalConcurrentTasks = globalSemaphore.getMaxPermits(); globalTaskUsageSamples.increment(); - if (globalSemaphore.getInUsePermits() >= maxGlobalConcurrentTasks) { + // A waiting task means every permit is effectively in use, even while a permit is handed to the next waiting task. + if (globalSemaphore.getInUsePermits() >= maxGlobalConcurrentTasks || globalSemaphore.getWaitingThreadCount() > 0) { globalTaskSlotsFullyUsedSamples.increment(); } @@ -513,6 +523,7 @@ private void evaluateAutoScheduling() { systemSchedulingSnapshot = systemSchedulingMetrics.captureSnapshot(maxGlobalConcurrentTasks, fractionOfSamplesWithAllGlobalTaskSlotsUsed, fractionOfGlobalTaskTimeSpentWaiting); lastSystemSchedulingSnapshotNanos = nowNanos; + balanceConcurrentTasks(nowNanos); } for (final SchedulingGeneration generation : schedulingGenerations.values()) { @@ -527,7 +538,8 @@ private void evaluateAutoScheduling() { continue; } - final ProcessorSchedulingSnapshot processorMeasurements = state.captureProcessorSchedulingSnapshot(nowNanos); + final boolean globalCapacityTestAllowed = concurrentTaskMoveCoordinator.isTestAllowed(state); + final ProcessorSchedulingSnapshot processorMeasurements = state.captureProcessorSchedulingSnapshot(nowNanos, globalCapacityTestAllowed); final long resetSequence = state.resetSequence.get(); final ProcessorSchedulingDecision decision = state.controller.evaluate(processorMeasurements, systemSchedulingSnapshot); logger.debug("Automatic scheduling evaluation for {}: processorMeasurements={}, systemMeasurements={}, decision={}", @@ -546,12 +558,38 @@ private void evaluateAutoScheduling() { state.applySettings(settings); startAutoWorkers(state, selectedConcurrentTasks); } + + concurrentTaskMoveCoordinator.recordDecision(state, decision.reason()); } } catch (final Throwable failure) { logger.error("Failed to evaluate automatic Processor scheduling", failure); } } + /** + * Updates each Processor's demand score and lets the coordinator end or start a concurrent task move. + */ + private void balanceConcurrentTasks(final long nowNanos) { + final List demands = new ArrayList<>(); + final Map statesByIdentifier = new HashMap<>(); + for (final SchedulingGeneration generation : schedulingGenerations.values()) { + final ProcessorAutoSchedulingState state = generation.getAutoSchedulingState(); + final ProcessorSchedulingSnapshot snapshot = state == null ? null : state.lastProcessorSchedulingSnapshot; + if (snapshot == null || generation.isStopped()) { + continue; + } + + final int maxConcurrentTasks = Math.min(state.maxConcurrentTasks, globalSemaphore.getMaxPermits()); + final ProcessorDemand demand = new ProcessorDemand(state.connectable.getIdentifier(), state.sourceProcessor, state.schedulingSettings.get().concurrentTasks(), + maxConcurrentTasks, state.controller.isConcurrencyComparisonActive(), state.controller.isIncreaseCoolingDown(nowNanos), snapshot); + state.demandScore = ConcurrentTaskMoveSelector.calculateScore(demand); + demands.add(demand); + statesByIdentifier.put(demand.identifier(), state); + } + + concurrentTaskMoveCoordinator.balance(demands, statesByIdentifier, systemSchedulingSnapshot.globalCapacityFull()); + } + private static CronExpression parseCronExpression(final String cronSchedule, final Object component) { try { return CronExpression.parse(cronSchedule); @@ -615,24 +653,7 @@ private void requestControllerReset(final SchedulingGeneration generation, final return; } - if (reason == AutoSchedulingResetReason.PROCESSOR_CONNECTIONS_CHANGED) { - generation.clearQueueRegistrations(); - registerQueueListeners(state.connectable, generation); - } - - state.resetMeasurementsAfterEvent(); - if (reason == AutoSchedulingResetReason.GLOBAL_CONCURRENT_TASK_LIMIT_CHANGED) { - final SchedulingSettings currentSettings = state.schedulingSettings.get(); - final int selectedConcurrentTasks = Math.min(currentSettings.concurrentTasks(), globalSemaphore.getMaxPermits()); - final SchedulingSettings clampedSettings = new SchedulingSettings(selectedConcurrentTasks, currentSettings.runDurationNanos()); - state.applySettings(clampedSettings); - startAutoWorkers(state, selectedConcurrentTasks); - } - - state.controller.reset(); - state.lastDecision = new ProcessorSchedulingDecision(state.schedulingSettings.get(), - ConcurrencyEvaluationState.USING_CURRENT_CONCURRENCY, - ProcessorSchedulingDecisionReason.PERFORMANCE_MEASUREMENTS_RESET, false); + resetController(generation, state, reason); }); } catch (final RejectedExecutionException e) { if (!shutdown.get()) { @@ -641,6 +662,30 @@ private void requestControllerReset(final SchedulingGeneration generation, final } } + /** + * Restarts a Processor's measurements after an event that makes them no longer comparable. Runs on the evaluation thread. + */ + private void resetController(final SchedulingGeneration generation, final ProcessorAutoSchedulingState state, final AutoSchedulingResetReason reason) { + if (reason == AutoSchedulingResetReason.PROCESSOR_CONNECTIONS_CHANGED) { + generation.clearQueueRegistrations(); + registerQueueListeners(state.connectable, generation); + } + + state.resetMeasurementsAfterEvent(); + if (reason == AutoSchedulingResetReason.GLOBAL_CONCURRENT_TASK_LIMIT_CHANGED) { + final SchedulingSettings currentSettings = state.schedulingSettings.get(); + final int selectedConcurrentTasks = Math.min(currentSettings.concurrentTasks(), globalSemaphore.getMaxPermits()); + final SchedulingSettings clampedSettings = new SchedulingSettings(selectedConcurrentTasks, currentSettings.runDurationNanos()); + state.applySettings(clampedSettings); + startAutoWorkers(state, selectedConcurrentTasks); + } + + state.controller.reset(); + state.lastDecision = new ProcessorSchedulingDecision(state.schedulingSettings.get(), + ConcurrencyEvaluationState.USING_CURRENT_CONCURRENCY, + ProcessorSchedulingDecisionReason.PERFORMANCE_MEASUREMENTS_RESET, false); + } + @Override public void setAdministrativeYieldDuration(final String duration) { this.adminYieldNanos = FormatUtils.getTimeDuration(duration, TimeUnit.NANOSECONDS); @@ -687,6 +732,12 @@ int getAutoSchedulingWaiterCount(final String componentIdentifier) { return generation == null ? 0 : generation.getWaiterCount(); } + ProcessorSchedulingMeasurements getAutoSchedulingMeasurements(final String componentIdentifier) { + final SchedulingGeneration generation = schedulingGenerations.get(componentIdentifier); + final ProcessorAutoSchedulingState state = generation == null ? null : generation.getAutoSchedulingState(); + return state == null ? null : state.measurements; + } + int getAutoSchedulingConcurrentTaskSlotWaiterCount(final String componentIdentifier) { final SchedulingGeneration generation = schedulingGenerations.get(componentIdentifier); return generation == null ? 0 : generation.getConcurrentTaskSlotWaiterCount(); @@ -966,7 +1017,7 @@ private ProcessorTaskWorker(final long identifier) { private record WaitingProcessorTask(Thread thread, ProcessorTaskWorker worker) { } - private class ProcessorAutoSchedulingState { + private class ProcessorAutoSchedulingState implements ConcurrentTaskMoveCoordinator.Participant { private final Connectable connectable; private final ConnectableTask connectableTask; private final LifecycleState lifecycleState; @@ -998,6 +1049,7 @@ private class ProcessorAutoSchedulingState { private volatile ProcessorSchedulingSnapshot lastProcessorSchedulingSnapshot; private volatile boolean previousPrimaryNode; private volatile boolean measurementsSupported; + private volatile double demandScore; private ProcessorAutoSchedulingState(final Connectable connectable, final ConnectableTask connectableTask, final LifecycleState lifecycleState, final SchedulingGeneration generation, @@ -1069,7 +1121,6 @@ boolean tryAcquireConcurrentTaskSlot(final SchedulingSettings expectedSettings) if (activeProcessorInvocations.compareAndSet(currentActiveProcessorInvocations, currentActiveProcessorInvocations + 1)) { if (schedulingSettings.get() == expectedSettings) { - measurements.recordInvocationStarted(); return true; } @@ -1081,7 +1132,6 @@ boolean tryAcquireConcurrentTaskSlot(final SchedulingSettings expectedSettings) } void releaseConcurrentTaskSlot() { - measurements.recordInvocationFinished(); activeProcessorInvocations.decrementAndGet(); generation.signalConcurrentTaskSlotChange(); } @@ -1153,7 +1203,7 @@ void recordReadiness(final long nowNanos) { measurements.recordReadiness(connectableTask.getReadinessOutcome(), measurementNanos); } - ProcessorSchedulingSnapshot captureProcessorSchedulingSnapshot(final long nowNanos) { + ProcessorSchedulingSnapshot captureProcessorSchedulingSnapshot(final long nowNanos, final boolean globalCapacityTestAllowed) { final SchedulingSettings currentSettings = schedulingSettings.get(); long inputQueueSize = 0L; for (final Connection connection : connectable.getIncomingConnections()) { @@ -1171,9 +1221,8 @@ ProcessorSchedulingSnapshot captureProcessorSchedulingSnapshot(final long nowNan final boolean sourceProcessorRecentlyReportedActivity = sourceProcessor && sourceDelayNanos == 0L && (activeProcessorInvocations.get() > 0 || lastSourceActivityNanos > 0L && nowNanos - lastSourceActivityNanos < TimeUnit.SECONDS.toNanos(2)); - final ProcessorSchedulingSnapshot snapshot = measurements.captureSnapshot(nowNanos, measurementWindowNanos, - currentSettings, connectableTask.isReady(), inputQueueHasFlowFiles, - inputQueueGrowth, sourceProcessorRecentlyReportedActivity, primaryNodeChanged); + final ProcessorSchedulingSnapshot snapshot = measurements.captureSnapshot(nowNanos, measurementWindowNanos, currentSettings, connectableTask.isReady(), + inputQueueHasFlowFiles, inputQueueSize, inputQueueGrowth, sourceProcessorRecentlyReportedActivity, primaryNodeChanged, globalCapacityTestAllowed); if (snapshot.committedFlowFiles() > 0L) { measurementsSupported = true; } @@ -1222,6 +1271,43 @@ void recordDecision(final ProcessorSchedulingDecision decision) { } } + @Override + public boolean isActive() { + return !generation.isStopped() && schedulingGenerations.get(connectable.getIdentifier()) == generation; + } + + @Override + public int getConcurrentTasks() { + return schedulingSettings.get().concurrentTasks(); + } + + @Override + public boolean isConcurrencyComparisonActive() { + return controller.isConcurrencyComparisonActive(); + } + + /** + * Changes the concurrent task setting for a concurrent task move and restarts the measurements without counting a failed test. + */ + @Override + public boolean changeConcurrentTasks(final int change) { + final SchedulingSettings currentSettings = schedulingSettings.get(); + final int updatedConcurrentTasks = currentSettings.concurrentTasks() + change; + if (!isActive() || updatedConcurrentTasks < 1 || updatedConcurrentTasks > Math.min(maxConcurrentTasks, globalSemaphore.getMaxPermits())) { + return false; + } + + applySettings(new SchedulingSettings(updatedConcurrentTasks, currentSettings.runDurationNanos())); + startAutoWorkers(this, updatedConcurrentTasks); + resetController(generation, this, AutoSchedulingResetReason.CONCURRENT_TASK_MOVED); + return true; + } + + @Override + public String toString() { + return connectable.toString(); + } + AutoSchedulingDiagnostics getDiagnostics() { final SchedulingSettings currentSettings = schedulingSettings.get(); final ProcessorSchedulingDecision currentDecision = lastDecision; @@ -1251,6 +1337,9 @@ AutoSchedulingDiagnostics getDiagnostics() { .setCollectingMeasurements(reason == ProcessorSchedulingDecisionReason.COLLECTING_COMPARISON_MEASUREMENTS) .setFlowFileMeasurementsAvailable(measurementsSupported) .setConcurrencyIncreaseExplanation(concurrencyUpdateStatus.explanation()) + .setLocalInputQueueCount(currentSnapshot == null ? 0L : currentSnapshot.localInputQueueCount()) + .setDemandScore(demandScore) + .setTaskMoveRole(concurrentTaskMoveCoordinator.getRole(this)) .build(); } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingDiagnostics.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingDiagnostics.java index 134b9da12737..5c026055d45d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingDiagnostics.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingDiagnostics.java @@ -30,6 +30,9 @@ public final class AutoSchedulingDiagnostics { private final boolean collectingMeasurements; private final boolean flowFileMeasurementsAvailable; private final String concurrencyIncreaseExplanation; + private final long localInputQueueCount; + private final double demandScore; + private final String taskMoveRole; private AutoSchedulingDiagnostics(final Builder builder) { executionMode = builder.executionMode; @@ -45,6 +48,9 @@ private AutoSchedulingDiagnostics(final Builder builder) { collectingMeasurements = builder.collectingMeasurements; flowFileMeasurementsAvailable = builder.flowFileMeasurementsAvailable; concurrencyIncreaseExplanation = builder.concurrencyIncreaseExplanation; + localInputQueueCount = builder.localInputQueueCount; + demandScore = builder.demandScore; + taskMoveRole = builder.taskMoveRole; } public static Builder createBuilder() { @@ -103,6 +109,18 @@ public String concurrencyIncreaseExplanation() { return concurrencyIncreaseExplanation; } + public long localInputQueueCount() { + return localInputQueueCount; + } + + public double demandScore() { + return demandScore; + } + + public String taskMoveRole() { + return taskMoveRole; + } + public static final class Builder { private String executionMode; private int maxConcurrentTasks; @@ -117,6 +135,9 @@ public static final class Builder { private boolean collectingMeasurements; private boolean flowFileMeasurementsAvailable; private String concurrencyIncreaseExplanation; + private long localInputQueueCount; + private double demandScore; + private String taskMoveRole; private Builder() { } @@ -186,6 +207,21 @@ public Builder setConcurrencyIncreaseExplanation(final String concurrencyIncreas return this; } + public Builder setLocalInputQueueCount(final long localInputQueueCount) { + this.localInputQueueCount = localInputQueueCount; + return this; + } + + public Builder setDemandScore(final double demandScore) { + this.demandScore = demandScore; + return this; + } + + public Builder setTaskMoveRole(final String taskMoveRole) { + this.taskMoveRole = taskMoveRole; + return this; + } + public AutoSchedulingDiagnostics build() { return new AutoSchedulingDiagnostics(this); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingResetReason.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingResetReason.java index 104d22cbde68..126049ef8c22 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingResetReason.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/AutoSchedulingResetReason.java @@ -18,5 +18,6 @@ public enum AutoSchedulingResetReason { PROCESSOR_CONNECTIONS_CHANGED, - GLOBAL_CONCURRENT_TASK_LIMIT_CHANGED + GLOBAL_CONCURRENT_TASK_LIMIT_CHANGED, + CONCURRENT_TASK_MOVED } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveCoordinator.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveCoordinator.java new file mode 100644 index 000000000000..4a54e89d6932 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveCoordinator.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.auto.ConcurrentTaskMoveSelector.ProcessorDemand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; + +/** + * Moves one concurrent task at a time from a donor to a receiver chosen by a {@link ConcurrentTaskMoveSelector}. The donor gives up its task + * before the receiver may test one more, so a move never adds concurrent tasks. The receiver keeps the task only when its test ends with + * higher throughput. Any other ending, including a reset during the test, rolls the receiver back and then gives the donor its task back. + * Must be updated from a single thread, but its read methods may be called from any thread. + */ +public class ConcurrentTaskMoveCoordinator { + private static final Logger logger = LoggerFactory.getLogger(ConcurrentTaskMoveCoordinator.class); + + private final ConcurrentTaskMoveSelector selector = new ConcurrentTaskMoveSelector(); + private volatile Move move; + + /** + * Ends a move whose receiver stopped or was reset mid-test, or selects a new move while global capacity is full. + * + * @param participants the participants for each of the demands, by identifier + */ + public void balance(final List demands, final Map participants, final boolean globalCapacityFull) { + final Move current = move; + if (current != null) { + if (!current.receiver.isActive() || current.testedConcurrentTasks > 0 && !current.receiver.isConcurrencyComparisonActive()) { + finish(current, false); + } + + return; + } + + selector.selectMove(demands, globalCapacityFull).ifPresent(selected -> { + final Participant receiver = participants.get(selected.receiverIdentifier()); + final Participant donor = participants.get(selected.donorIdentifier()); + if (donor.changeConcurrentTasks(-1)) { + move = new Move(receiver, donor); + logger.info("Moving one concurrent task from {} to {} because its queued work is much larger", donor, receiver); + } + }); + } + + /** + * @return whether the participant may test one more concurrent task even though global capacity is full + */ + public boolean isTestAllowed(final Participant participant) { + final Move current = move; + return current != null && current.receiver == participant && current.testedConcurrentTasks == 0; + } + + /** + * Records the setting the receiver tests with the moved task, and ends the move once the receiver is not testing it. + */ + public void recordDecision(final Participant participant, final ProcessorSchedulingDecisionReason reason) { + final Move current = move; + if (current == null || current.receiver != participant) { + return; + } + + if (current.testedConcurrentTasks == 0 && reason == ProcessorSchedulingDecisionReason.TESTING_HIGHER_CONCURRENCY) { + current.testedConcurrentTasks = participant.getConcurrentTasks(); + } else if (!participant.isConcurrencyComparisonActive()) { + finish(current, current.testedConcurrentTasks > 0 && reason == ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_INCREASED_THROUGHPUT); + } + } + + /** + * @return {@code RECEIVER} or {@code DONOR} while the participant takes part in a move, otherwise {@code null} + */ + public String getRole(final Participant participant) { + final Move current = move; + if (current != null && current.receiver == participant) { + return "RECEIVER"; + } + + return current != null && current.donor == participant ? "DONOR" : null; + } + + private void finish(final Move current, final boolean throughputIncreased) { + move = null; + if (throughputIncreased) { + return; + } + + // A test that ended without higher throughput has already lowered the receiver, but a reset leaves it on the tested setting. + // Lowering the receiver before raising the donor keeps the total from ever exceeding what it was before the move. + if (current.testedConcurrentTasks > 0 && current.receiver.isActive() && current.receiver.getConcurrentTasks() >= current.testedConcurrentTasks) { + current.receiver.changeConcurrentTasks(-1); + } + + current.donor.changeConcurrentTasks(1); + } + + /** + * A Processor that can give or receive a concurrent task. + */ + public interface Participant { + /** + * @return whether the Processor is still scheduled in the same run + */ + boolean isActive(); + + int getConcurrentTasks(); + + boolean isConcurrencyComparisonActive(); + + /** + * Changes the concurrent task setting by the given amount if the Processor is active and the result is within its limits. + * + * @return whether the setting was changed + */ + boolean changeConcurrentTasks(int change); + } + + private static class Move { + private final Participant receiver; + private final Participant donor; + private volatile int testedConcurrentTasks; + + private Move(final Participant receiver, final Participant donor) { + this.receiver = receiver; + this.donor = donor; + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveSelector.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveSelector.java new file mode 100644 index 000000000000..0f9cde05deba --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveSelector.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingSnapshot.RecentDemand; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** + * While the global concurrent task limit is fully used, chooses when one concurrent task should move from a Processor with little + * queued work to a Processor with much more. A Processor's demand score is its queued FlowFiles divided by its recent rate of committed + * input FlowFiles: roughly how many seconds its backlog would take to clear. A move is proposed only after the same pair has shown a + * clear difference in score for several evaluations in a row. Not thread-safe. + */ +public class ConcurrentTaskMoveSelector { + static final long MIN_MEASUREMENT_NANOS = TimeUnit.SECONDS.toNanos(6); + static final double MAX_SCORE = 60D; + static final double MIN_RECEIVER_SCORE = 10D; + static final double SOURCE_SCORE = MIN_RECEIVER_SCORE; + static final int SUSTAINED_EVALUATIONS = 3; + private static final double MIN_SCORE_RATIO = 2D; + private static final double MIN_SCORE_DIFFERENCE = 5D; + private static final double AVERAGING_SECONDS = RecentDemand.AVERAGING_NANOS / (double) TimeUnit.SECONDS.toNanos(1); + + private TaskMove candidateMove; + private int consecutiveEvaluations; + + public Optional selectMove(final List demands, final boolean globalCapacityFull) { + if (!globalCapacityFull) { + candidateMove = null; + consecutiveEvaluations = 0; + return Optional.empty(); + } + + ProcessorDemand receiver = null; + double receiverScore = 0D; + for (final ProcessorDemand demand : demands) { + final double score = calculateScore(demand); + if (isEligibleReceiver(demand, score) && (receiver == null || score > receiverScore)) { + receiver = demand; + receiverScore = score; + } + } + + ProcessorDemand donor = null; + double donorScore = 0D; + for (final ProcessorDemand demand : demands) { + if (demand == receiver || demand.source() || demand.concurrentTasks() <= 1 || demand.concurrencyComparisonActive()) { + continue; + } + + final double score = calculateScore(demand); + if (donor == null || score < donorScore || score == donorScore && demand.concurrentTasks() > donor.concurrentTasks()) { + donor = demand; + donorScore = score; + } + } + + if (receiver == null || donor == null || receiverScore < donorScore * MIN_SCORE_RATIO || receiverScore < donorScore + MIN_SCORE_DIFFERENCE) { + candidateMove = null; + consecutiveEvaluations = 0; + return Optional.empty(); + } + + final TaskMove move = new TaskMove(receiver.identifier(), donor.identifier()); + consecutiveEvaluations = move.equals(candidateMove) ? consecutiveEvaluations + 1 : 1; + candidateMove = move; + if (consecutiveEvaluations < SUSTAINED_EVALUATIONS) { + return Optional.empty(); + } + + // The difference must be sustained again before the next move. + candidateMove = null; + consecutiveEvaluations = 0; + return Optional.of(move); + } + + public static double calculateScore(final ProcessorDemand demand) { + final ProcessorSchedulingSnapshot snapshot = demand.snapshot(); + if (demand.source()) { + final boolean activeAndBusy = snapshot.processorReady() && snapshot.sourceProcessorRecentlyReportedActivity() + && snapshot.concurrentTaskUtilization() >= StandardAutoSchedulingController.MIN_UTILIZATION_FOR_INCREASE; + return activeAndBusy ? SOURCE_SCORE : 0D; + } + + if (snapshot.localInputQueueCount() == 0L) { + return 0D; + } + + final RecentDemand recent = snapshot.recentDemand(); + final boolean committedInput = isRecent(recent.committedInputFlowFilesPerSecond()); + final boolean notProgressing = isRecent(recent.completedInvocationsPerSecond()) && !committedInput; + if (recent.measuredNanos() >= MIN_MEASUREMENT_NANOS && (isFailing(recent) || notProgressing)) { + // A Processor that fails or commits nothing gains nothing from its extra tasks, so it is the first to give one up. + return 0D; + } + + return committedInput ? Math.min(MAX_SCORE, snapshot.localInputQueueCount() / recent.committedInputFlowFilesPerSecond()) : MAX_SCORE; + } + + private static boolean isEligibleReceiver(final ProcessorDemand demand, final double score) { + final RecentDemand recent = demand.snapshot().recentDemand(); + return score >= MIN_RECEIVER_SCORE + && demand.snapshot().processorReady() + && demand.concurrentTasks() < demand.maxConcurrentTasks() + && !isFailing(recent) + && recent.measuredNanos() >= MIN_MEASUREMENT_NANOS + && !demand.concurrencyComparisonActive() + && !demand.increaseCoolingDown() + && (demand.source() || isRecent(recent.committedInputFlowFilesPerSecond())); + } + + private static boolean isFailing(final RecentDemand recent) { + return isRecent(recent.failedInvocationsPerSecond()) + && recent.failedInvocationsPerSecond() >= recent.completedInvocationsPerSecond() * StandardAutoSchedulingController.MAX_FAILED_INVOCATION_FRACTION; + } + + /** + * @return whether a rate amounts to at least one occurrence during the averaging period + */ + private static boolean isRecent(final double perSecond) { + return perSecond * AVERAGING_SECONDS >= 1D; + } + + /** + * @param source whether the Processor has no input queue of its own + * @param maxConcurrentTasks the most concurrent tasks the Processor may use + * @param increaseCoolingDown whether the Processor must wait before testing another increase + */ + public record ProcessorDemand(String identifier, boolean source, int concurrentTasks, int maxConcurrentTasks, boolean concurrencyComparisonActive, + boolean increaseCoolingDown, ProcessorSchedulingSnapshot snapshot) { + } + + public record TaskMove(String receiverIdentifier, String donorIdentifier) { + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurements.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurements.java index 2436bdff1a4e..30388ce7b4ff 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurements.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurements.java @@ -28,19 +28,22 @@ public class ProcessorSchedulingMeasurements { private final LongAdder committedFlowFiles = new LongAdder(); + private final LongAdder committedInputFlowFiles = new LongAdder(); private final LongAdder completedInvocations = new LongAdder(); private final LongAdder failedInvocations = new LongAdder(); private final LongAdder processorInvocationNanos = new LongAdder(); private final LongSupplier nanoTimeSupplier; - private final Object activeInvocationLock = new Object(); - private int activeInvocations; - private long activeInvocationChangeNanos; - private long activeInvocationWindowStartNanos; - private long activeInvocationNanos; + private final Object busyTaskLock = new Object(); + private int busyTasks; + private long busyTaskChangeNanos; + private long busyTaskWindowStartNanos; + private long busyTaskNanos; private long previousCommittedFlowFiles; + private long previousCommittedInputFlowFiles; private long previousCompletedInvocations; private long previousFailedInvocations; private long previousProcessorInvocationNanos; + private ProcessorSchedulingSnapshot.RecentDemand recentDemand = ProcessorSchedulingSnapshot.RecentDemand.NONE; private long backpressureNanos; private long processorYieldNanos; @@ -50,8 +53,8 @@ public ProcessorSchedulingMeasurements() { ProcessorSchedulingMeasurements(final LongSupplier nanoTimeSupplier) { this.nanoTimeSupplier = nanoTimeSupplier; - activeInvocationChangeNanos = nanoTimeSupplier.getAsLong(); - activeInvocationWindowStartNanos = activeInvocationChangeNanos; + busyTaskChangeNanos = nanoTimeSupplier.getAsLong(); + busyTaskWindowStartNanos = busyTaskChangeNanos; } public InvocationObserver createInvocationObserver() { @@ -62,17 +65,21 @@ public void recordInvocationDuration(final long nanos) { processorInvocationNanos.add(Math.max(0L, nanos)); } - public void recordInvocationStarted() { - synchronized (activeInvocationLock) { - accumulateActiveInvocationNanos(); - activeInvocations++; + /** + * Records that a concurrent task has become busy. A task is busy from the time it starts waiting for a global permit + * until its invocation finishes, because a task waiting for a permit is ready to work. + */ + public void recordTaskBusy() { + synchronized (busyTaskLock) { + accumulateBusyTaskNanos(); + busyTasks++; } } - public void recordInvocationFinished() { - synchronized (activeInvocationLock) { - accumulateActiveInvocationNanos(); - activeInvocations--; + public void recordTaskIdle() { + synchronized (busyTaskLock) { + accumulateBusyTaskNanos(); + busyTasks--; } } @@ -85,38 +92,45 @@ public void recordReadiness(final InvocationOutcome readinessOutcome, final long } /** - * Captures the measurements recorded since the previous snapshot. Concurrent task utilization is the total time spent in - * invocations since the previous snapshot, including invocations that are still running, divided by the time available - * to the allowed concurrent tasks over that same period. Utilization periods begin and end at times read while holding the - * lock that also guards invocation start and finish, so each moment of invocation time belongs to exactly one snapshot even - * when the caller's timestamp was read before an invocation started. + * Captures the measurements recorded since the previous snapshot. Concurrent task utilization is the total time that concurrent + * tasks were busy since the previous snapshot, including tasks that are still busy, divided by the time available to the allowed + * concurrent tasks over that same period. Utilization periods begin and end at times read while holding the lock that also guards + * changes in the number of busy tasks, so each moment of busy time belongs to exactly one snapshot even when the caller's timestamp + * was read before a task became busy. */ public ProcessorSchedulingSnapshot captureSnapshot(final long timestampNanos, final long measurementWindowNanos, final SchedulingSettings settings, - final boolean processorReady, final boolean inputQueueHasFlowFiles, final double inputQueueGrowth, - final boolean sourceProcessorRecentlyReportedActivity, final boolean primaryNodeChanged) { + final boolean processorReady, final boolean inputQueueHasFlowFiles, final long localInputQueueCount, + final double inputQueueGrowth, final boolean sourceProcessorRecentlyReportedActivity, + final boolean primaryNodeChanged, final boolean globalCapacityTestAllowed) { final long committedFlowFileCount = committedFlowFiles.sum(); + final long committedInputFlowFileCount = committedInputFlowFiles.sum(); final long completedInvocationCount = completedInvocations.sum(); final long failedInvocationCount = failedInvocations.sum(); final long processorInvocationDurationNanos = processorInvocationNanos.sum(); - final long windowActiveInvocationNanos; + final long windowBusyTaskNanos; final long utilizationWindowNanos; - synchronized (activeInvocationLock) { - accumulateActiveInvocationNanos(); - windowActiveInvocationNanos = activeInvocationNanos; - utilizationWindowNanos = activeInvocationChangeNanos - activeInvocationWindowStartNanos; - activeInvocationNanos = 0L; - activeInvocationWindowStartNanos = activeInvocationChangeNanos; + synchronized (busyTaskLock) { + accumulateBusyTaskNanos(); + windowBusyTaskNanos = busyTaskNanos; + utilizationWindowNanos = busyTaskChangeNanos - busyTaskWindowStartNanos; + busyTaskNanos = 0L; + busyTaskWindowStartNanos = busyTaskChangeNanos; } + final long completedInvocationDelta = completedInvocationCount - previousCompletedInvocations; + final long failedInvocationDelta = failedInvocationCount - previousFailedInvocations; + recentDemand = recentDemand.add(measurementWindowNanos, committedInputFlowFileCount - previousCommittedInputFlowFiles, completedInvocationDelta, + failedInvocationDelta); + final double concurrentTaskUtilization = Math.min(1D, - windowActiveInvocationNanos / ((double) Math.max(1L, utilizationWindowNanos) * settings.concurrentTasks())); + windowBusyTaskNanos / ((double) Math.max(1L, utilizationWindowNanos) * settings.concurrentTasks())); final ProcessorSchedulingSnapshot snapshot = ProcessorSchedulingSnapshot.createBuilder() .setTimestampNanos(timestampNanos) .setMeasurementWindowNanos(measurementWindowNanos) .setAppliedSettings(settings) .setCommittedFlowFiles(committedFlowFileCount - previousCommittedFlowFiles) - .setCompletedInvocations(completedInvocationCount - previousCompletedInvocations) - .setFailedInvocations(failedInvocationCount - previousFailedInvocations) + .setCompletedInvocations(completedInvocationDelta) + .setFailedInvocations(failedInvocationDelta) .setProcessorInvocationNanos(processorInvocationDurationNanos - previousProcessorInvocationNanos) .setBackpressureNanos(backpressureNanos) .setProcessorYieldNanos(processorYieldNanos) @@ -126,8 +140,12 @@ public ProcessorSchedulingSnapshot captureSnapshot(final long timestampNanos, fi .setInputQueueGrowth(inputQueueGrowth) .setSourceProcessorRecentlyReportedActivity(sourceProcessorRecentlyReportedActivity) .setPrimaryNodeChanged(primaryNodeChanged) + .setLocalInputQueueCount(localInputQueueCount) + .setRecentDemand(recentDemand) + .setGlobalCapacityTestAllowed(globalCapacityTestAllowed) .build(); previousCommittedFlowFiles = committedFlowFileCount; + previousCommittedInputFlowFiles = committedInputFlowFileCount; previousCompletedInvocations = completedInvocationCount; previousFailedInvocations = failedInvocationCount; previousProcessorInvocationNanos = processorInvocationDurationNanos; @@ -137,13 +155,13 @@ public ProcessorSchedulingSnapshot captureSnapshot(final long timestampNanos, fi } /** - * Adds the invocation time since the last change. Must be called while holding the active invocation lock; reading the + * Adds the busy time since the last change. Must be called while holding the busy task lock; reading the * clock under the lock keeps the change timestamp from moving backward. */ - private void accumulateActiveInvocationNanos() { + private void accumulateBusyTaskNanos() { final long nowNanos = nanoTimeSupplier.getAsLong(); - activeInvocationNanos += activeInvocations * (nowNanos - activeInvocationChangeNanos); - activeInvocationChangeNanos = nowNanos; + busyTaskNanos += busyTasks * (nowNanos - busyTaskChangeNanos); + busyTaskChangeNanos = nowNanos; } private void recordCompletedInvocation(final InvocationOutcome outcome) { @@ -154,6 +172,7 @@ private void recordCompletedInvocation(final InvocationOutcome outcome) { } private long recordCommittedFlowFiles(final CommittedSchedulingWork committedWork) { + committedInputFlowFiles.add(committedWork.inputFlowFiles()); // Count committed input FlowFiles when present, otherwise count committed produced FlowFiles for source processors. final long count = committedWork.inputFlowFiles() > 0L ? committedWork.inputFlowFiles() : committedWork.producedFlowFiles(); committedFlowFiles.add(count); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingSnapshot.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingSnapshot.java index c91c711b8c4c..482d8fd3315d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingSnapshot.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingSnapshot.java @@ -18,6 +18,8 @@ import org.apache.nifi.controller.scheduling.SchedulingSettings; +import java.util.concurrent.TimeUnit; + public final class ProcessorSchedulingSnapshot { private final long timestampNanos; private final long measurementWindowNanos; @@ -34,6 +36,9 @@ public final class ProcessorSchedulingSnapshot { private final double inputQueueGrowth; private final boolean sourceProcessorRecentlyReportedActivity; private final boolean primaryNodeChanged; + private final long localInputQueueCount; + private final RecentDemand recentDemand; + private final boolean globalCapacityTestAllowed; private ProcessorSchedulingSnapshot(final Builder builder) { timestampNanos = builder.timestampNanos; @@ -51,6 +56,9 @@ private ProcessorSchedulingSnapshot(final Builder builder) { inputQueueGrowth = builder.inputQueueGrowth; sourceProcessorRecentlyReportedActivity = builder.sourceProcessorRecentlyReportedActivity; primaryNodeChanged = builder.primaryNodeChanged; + localInputQueueCount = builder.localInputQueueCount; + recentDemand = builder.recentDemand; + globalCapacityTestAllowed = builder.globalCapacityTestAllowed; } public static Builder createBuilder() { @@ -117,6 +125,45 @@ public boolean primaryNodeChanged() { return primaryNodeChanged; } + public long localInputQueueCount() { + return localInputQueueCount; + } + + public RecentDemand recentDemand() { + return recentDemand; + } + + /** + * @return whether this evaluation may test one more concurrent task even though global capacity is full + */ + public boolean globalCapacityTestAllowed() { + return globalCapacityTestAllowed; + } + + /** + * Recent per-second rates, averaged over all measurements until {@link #AVERAGING_NANOS} have been measured and then weighted toward + * the most recent {@link #AVERAGING_NANOS}. + * + * @param committedInputFlowFilesPerSecond input FlowFiles committed per second + * @param completedInvocationsPerSecond invocations completed per second + * @param failedInvocationsPerSecond invocations failed per second + * @param measuredNanos total time measured + */ + public record RecentDemand(double committedInputFlowFilesPerSecond, double completedInvocationsPerSecond, double failedInvocationsPerSecond, + long measuredNanos) { + public static final long AVERAGING_NANOS = TimeUnit.SECONDS.toNanos(10); + public static final RecentDemand NONE = new RecentDemand(0D, 0D, 0D, 0L); + + public RecentDemand add(final long windowNanos, final long committedInputFlowFiles, final long completedInvocations, final long failedInvocations) { + final long totalNanos = measuredNanos + windowNanos; + final double weight = Math.min(1D, windowNanos / (double) Math.max(1L, Math.min(totalNanos, AVERAGING_NANOS))); + final double windowSeconds = Math.max(1L, windowNanos) / (double) TimeUnit.SECONDS.toNanos(1); + return new RecentDemand(committedInputFlowFilesPerSecond + weight * (committedInputFlowFiles / windowSeconds - committedInputFlowFilesPerSecond), + completedInvocationsPerSecond + weight * (completedInvocations / windowSeconds - completedInvocationsPerSecond), + failedInvocationsPerSecond + weight * (failedInvocations / windowSeconds - failedInvocationsPerSecond), totalNanos); + } + } + public static final class Builder { private long timestampNanos; private long measurementWindowNanos; @@ -133,6 +180,9 @@ public static final class Builder { private double inputQueueGrowth; private boolean sourceProcessorRecentlyReportedActivity; private boolean primaryNodeChanged; + private long localInputQueueCount; + private RecentDemand recentDemand = RecentDemand.NONE; + private boolean globalCapacityTestAllowed; private Builder() { } @@ -212,6 +262,21 @@ public Builder setPrimaryNodeChanged(final boolean primaryNodeChanged) { return this; } + public Builder setLocalInputQueueCount(final long localInputQueueCount) { + this.localInputQueueCount = localInputQueueCount; + return this; + } + + public Builder setRecentDemand(final RecentDemand recentDemand) { + this.recentDemand = recentDemand; + return this; + } + + public Builder setGlobalCapacityTestAllowed(final boolean globalCapacityTestAllowed) { + this.globalCapacityTestAllowed = globalCapacityTestAllowed; + return this; + } + public ProcessorSchedulingSnapshot build() { return new ProcessorSchedulingSnapshot(this); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingController.java index 34069b4db581..e109f1611a61 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingController.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingController.java @@ -32,7 +32,8 @@ *

When there is sustained work and the existing calls are busy, the controller tests a higher number of * concurrent calls. Each test adds one call, and the higher number is kept only when it produces a clear increase * in completed work. No increase exceeds the processor maximum or the global concurrent task limit. An unsuccessful - * increase waits before it is tried again.

+ * increase waits before it is tried again. While the global concurrent task limit is fully used, an increase is tested only when + * the snapshot allows it, because the scheduling agent chose this processor to take a task from one with far less queued work.

* *

The controller also looks for a lower number of calls. If the processor is mostly idle or blocked for several * seconds, it removes one unused call immediately. At regular intervals it tests one fewer call even when the @@ -53,7 +54,8 @@ public class StandardAutoSchedulingController { private static final long REDUCTION_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); private static final long UNUSED_TASK_REDUCTION_NANOS = TimeUnit.SECONDS.toNanos(3); private static final double THROUGHPUT_TOLERANCE = 0.05D; - private static final double MAX_FAILED_INVOCATION_FRACTION = 0.10D; + static final double MAX_FAILED_INVOCATION_FRACTION = 0.10D; + static final double MIN_UTILIZATION_FOR_INCREASE = 0.75D; private final int maxConcurrentTasks; private final long runDurationNanos; @@ -175,6 +177,14 @@ public ConcurrencyUpdateStatus getConcurrencyUpdateStatus() { return concurrencyUpdateStatus; } + public boolean isConcurrencyComparisonActive() { + return concurrencyComparison != null; + } + + public boolean isIncreaseCoolingDown(final long timestampNanos) { + return timestampNanos < nextIncreaseNanos; + } + /** * Chooses a measurement window long enough for each concurrent task to complete about * {@value #COMPARISON_INVOCATIONS_PER_TASK} invocations, so that partial invocations at the edges of the window @@ -320,7 +330,7 @@ private ConcurrencyUpdateReason findIncreaseBlocker(final int concurrentTasks, f if (systemMeasurements.garbageCollectionTimeAboveLimit()) { return ConcurrencyUpdateReason.GARBAGE_COLLECTION_TIME_TOO_HIGH; } - if (systemMeasurements.fractionOfSamplesWithAllGlobalTaskSlotsUsed() >= 0.90D && systemMeasurements.fractionOfGlobalTaskTimeSpentWaiting() > 0.10D) { + if (systemMeasurements.globalCapacityFull() && !processorMeasurements.globalCapacityTestAllowed()) { return ConcurrencyUpdateReason.GLOBAL_CAPACITY_FULL; } if (!processorMeasurements.inputQueueHasFlowFiles() && !processorMeasurements.sourceProcessorRecentlyReportedActivity()) { @@ -335,7 +345,7 @@ private ConcurrencyUpdateReason findIncreaseBlocker(final int concurrentTasks, f if (isProcessorYielding(processorMeasurements)) { return ConcurrencyUpdateReason.PROCESSOR_YIELDING; } - if (processorMeasurements.concurrentTaskUtilization() < 0.75D) { + if (processorMeasurements.concurrentTaskUtilization() < MIN_UTILIZATION_FOR_INCREASE) { return ConcurrencyUpdateReason.CONCURRENT_TASKS_UNDERUSED; } if (processorMeasurements.timestampNanos() < nextIncreaseNanos) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingSnapshot.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingSnapshot.java index a3bcae852263..8185c003c7db 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingSnapshot.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/auto/SystemSchedulingSnapshot.java @@ -79,6 +79,14 @@ public boolean garbageCollectionTimeAboveLimit() { return garbageCollectionTimeAboveLimit; } + /** + * @return whether nearly every sample found all global task slots in use while tasks also spent a meaningful share of their time + * waiting for a slot + */ + public boolean globalCapacityFull() { + return fractionOfSamplesWithAllGlobalTaskSlotsUsed >= 0.90D && fractionOfGlobalTaskTimeSpentWaiting > 0.10D; + } + public static final class Builder { private boolean cpuLoadAvailable; private double cpuLoad; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveCoordinatorTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveCoordinatorTest.java new file mode 100644 index 000000000000..0b39533a8fcd --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveCoordinatorTest.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.SchedulingSettings; +import org.apache.nifi.controller.scheduling.auto.ConcurrentTaskMoveSelector.ProcessorDemand; +import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingSnapshot.RecentDemand; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ConcurrentTaskMoveCoordinatorTest { + private final ConcurrentTaskMoveCoordinator coordinator = new ConcurrentTaskMoveCoordinator(); + private final TestParticipant receiver = new TestParticipant(1); + private final TestParticipant donor = new TestParticipant(6); + private final Map participants = Map.of("receiver", receiver, "donor", donor); + private int maxTotalConcurrentTasks; + + @ParameterizedTest + @EnumSource(MoveEnding.class) + void testMoveNeverAddsConcurrentTasks(final MoveEnding moveEnding) { + selectMove(); + // The donor has already given up its task when the receiver is first allowed to test one more. + assertEquals(5, donor.concurrentTasks); + assertTrue(coordinator.isTestAllowed(receiver)); + assertFalse(coordinator.isTestAllowed(donor)); + assertEquals("RECEIVER", coordinator.getRole(receiver)); + assertEquals("DONOR", coordinator.getRole(donor)); + + if (moveEnding == MoveEnding.RECEIVER_DECLINES) { + coordinator.recordDecision(receiver, ProcessorSchedulingDecisionReason.GLOBAL_CAPACITY_FULL); + } else { + startReceiverTest(); + assertFalse(coordinator.isTestAllowed(receiver)); + // A further selection is ignored while the move is in progress. + balance(); + assertEquals(5, donor.concurrentTasks); + + switch (moveEnding) { + case THROUGHPUT_INCREASED -> endReceiverTest(2, ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_INCREASED_THROUGHPUT); + case THROUGHPUT_NOT_INCREASED -> endReceiverTest(1, ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_DID_NOT_INCREASE_THROUGHPUT); + // A reset ends the test without undoing the receiver's extra task, and either the next balance or the receiver's next + // evaluation may be the first to notice. + case RESET_NOTICED_BY_BALANCE -> { + receiver.comparisonActive = false; + balance(); + } + case RESET_NOTICED_BY_EVALUATION -> endReceiverTest(2, ProcessorSchedulingDecisionReason.PERFORMANCE_MEASUREMENTS_RESET); + case RECEIVER_STOPPED -> { + receiver.active = false; + balance(); + } + default -> throw new IllegalStateException(moveEnding.name()); + } + } + + // Only a test that proved higher throughput lets the receiver keep the task. + final boolean receiverKeepsTask = moveEnding == MoveEnding.THROUGHPUT_INCREASED; + if (moveEnding != MoveEnding.RECEIVER_STOPPED) { + assertEquals(receiverKeepsTask ? 2 : 1, receiver.concurrentTasks); + } + + assertEquals(receiverKeepsTask ? 5 : 6, donor.concurrentTasks); + assertTrue(maxTotalConcurrentTasks <= 7, "Total concurrent tasks reached " + maxTotalConcurrentTasks); + assertNull(coordinator.getRole(receiver)); + assertNull(coordinator.getRole(donor)); + assertFalse(coordinator.isTestAllowed(receiver)); + } + + @Test + void testNoMoveWhenDonorCannotGiveUpTask() { + donor.active = false; + selectMove(); + assertEquals(6, donor.concurrentTasks); + assertFalse(coordinator.isTestAllowed(receiver)); + assertNull(coordinator.getRole(receiver)); + + // A donor that stops during a move does not get the task back. + donor.active = true; + selectMove(); + assertEquals(5, donor.concurrentTasks); + donor.active = false; + coordinator.recordDecision(receiver, ProcessorSchedulingDecisionReason.GLOBAL_CAPACITY_FULL); + assertEquals(5, donor.concurrentTasks); + assertNull(coordinator.getRole(donor)); + } + + private void selectMove() { + for (int evaluation = 0; evaluation < ConcurrentTaskMoveSelector.SUSTAINED_EVALUATIONS; evaluation++) { + balance(); + } + } + + private void balance() { + coordinator.balance(List.of(demand("receiver", receiver, 10_000), demand("donor", donor, 1)), participants, true); + } + + private void startReceiverTest() { + receiver.concurrentTasks = 2; + receiver.comparisonActive = true; + coordinator.recordDecision(receiver, ProcessorSchedulingDecisionReason.TESTING_HIGHER_CONCURRENCY); + } + + private void endReceiverTest(final int concurrentTasks, final ProcessorSchedulingDecisionReason reason) { + receiver.concurrentTasks = concurrentTasks; + receiver.comparisonActive = false; + coordinator.recordDecision(receiver, reason); + } + + private static ProcessorDemand demand(final String identifier, final TestParticipant participant, final long queuedFlowFiles) { + final ProcessorSchedulingSnapshot snapshot = ProcessorSchedulingSnapshot.createBuilder() + .setAppliedSettings(new SchedulingSettings(participant.concurrentTasks, 0L)) + .setProcessorReady(true) + .setConcurrentTaskUtilization(1D) + .setLocalInputQueueCount(queuedFlowFiles) + .setRecentDemand(new RecentDemand(100, 10, 0, TimeUnit.SECONDS.toNanos(10))) + .build(); + return new ProcessorDemand(identifier, false, participant.concurrentTasks, 12, participant.comparisonActive, false, snapshot); + } + + private enum MoveEnding { + THROUGHPUT_INCREASED, + THROUGHPUT_NOT_INCREASED, + RESET_NOTICED_BY_BALANCE, + RESET_NOTICED_BY_EVALUATION, + RECEIVER_STOPPED, + RECEIVER_DECLINES + } + + /** + * Records the largest total of the active participants' settings reached by any change the coordinator makes. + */ + private class TestParticipant implements ConcurrentTaskMoveCoordinator.Participant { + private int concurrentTasks; + private boolean active = true; + private boolean comparisonActive; + + private TestParticipant(final int concurrentTasks) { + this.concurrentTasks = concurrentTasks; + } + + @Override + public boolean isActive() { + return active; + } + + @Override + public int getConcurrentTasks() { + return concurrentTasks; + } + + @Override + public boolean isConcurrencyComparisonActive() { + return comparisonActive; + } + + @Override + public boolean changeConcurrentTasks(final int change) { + if (!active || concurrentTasks + change < 1) { + return false; + } + + concurrentTasks += change; + maxTotalConcurrentTasks = Math.max(maxTotalConcurrentTasks, receiver.getActiveConcurrentTasks() + donor.getActiveConcurrentTasks()); + return true; + } + + private int getActiveConcurrentTasks() { + return active ? concurrentTasks : 0; + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveSelectorTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveSelectorTest.java new file mode 100644 index 000000000000..326fe3383740 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ConcurrentTaskMoveSelectorTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.scheduling.auto; + +import org.apache.nifi.controller.scheduling.SchedulingSettings; +import org.apache.nifi.controller.scheduling.auto.ConcurrentTaskMoveSelector.ProcessorDemand; +import org.apache.nifi.controller.scheduling.auto.ConcurrentTaskMoveSelector.TaskMove; +import org.apache.nifi.controller.scheduling.auto.ProcessorSchedulingSnapshot.RecentDemand; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ConcurrentTaskMoveSelectorTest { + private static final long MEASURED_NANOS = TimeUnit.SECONDS.toNanos(10); + + @Test + void testMoveRequiresSustainedMarginBetweenHighestReceiverAndLowestDonor() { + // At 100 committed FlowFiles per second: 10,000 queued is capped at 60 seconds, 50 queued is 0.5 seconds, and 200 queued is 2 seconds. + final List demands = List.of(queueConsumer("other-donor", 6, 200, 100), queueConsumer("receiver", 1, 10_000, 100), + queueConsumer("lowest-donor", 6, 50, 100)); + final ConcurrentTaskMoveSelector selector = new ConcurrentTaskMoveSelector(); + assertTrue(selectSustainedMove(selector, demands, false).isEmpty()); + assertTrue(selector.selectMove(demands, true).isEmpty()); + assertTrue(selector.selectMove(demands, true).isEmpty()); + assertEquals(Optional.of(new TaskMove("receiver", "lowest-donor")), selector.selectMove(demands, true)); + + // After a move is returned, the margin must be sustained again before the next move. + assertTrue(selector.selectMove(demands, true).isEmpty()); + assertTrue(selector.selectMove(demands, true).isEmpty()); + assertTrue(selector.selectMove(demands, true).isPresent()); + + // 12 seconds against 7 seconds meets the minimum difference but not twice the donor's score. + assertTrue(selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(queueConsumer("receiver", 1, 1_200, 100), queueConsumer("donor", 6, 700, 100)), true) + .isEmpty()); + // Nearly empty queues never reach the minimum receiver score, so they do not pass tasks between themselves. + assertTrue(selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(queueConsumer("receiver", 1, 90, 100), queueConsumer("donor", 6, 1, 100)), true) + .isEmpty()); + // Among donors with the same score, the donor with the most tasks gives one up. + final List equalDonors = List.of(queueConsumer("receiver", 1, 10_000, 100), queueConsumer("fewer", 3, 0, 100), + queueConsumer("more", 8, 0, 100)); + assertEquals(Optional.of(new TaskMove("receiver", "more")), selectSustainedMove(new ConcurrentTaskMoveSelector(), equalDonors, true)); + } + + @Test + void testSourceTakesTasksOnlyFromProcessorsWithoutRealBacklog() { + final ProcessorDemand activeSource = source(true, true, 1D); + assertEquals(ConcurrentTaskMoveSelector.SOURCE_SCORE, ConcurrentTaskMoveSelector.calculateScore(activeSource)); + assertEquals(0D, ConcurrentTaskMoveSelector.calculateScore(source(false, true, 1D))); + assertEquals(0D, ConcurrentTaskMoveSelector.calculateScore(source(true, false, 1D))); + assertEquals(0D, ConcurrentTaskMoveSelector.calculateScore(source(true, true, 0.5D))); + + assertEquals(Optional.of(new TaskMove("source", "donor")), + selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(activeSource, queueConsumer("donor", 6, 10, 100)), true)); + // 20 queued at 1 per second is 20 seconds of work, a real backlog. + assertTrue(selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(activeSource, queueConsumer("donor", 6, 20, 1)), true).isEmpty()); + // A source is never a donor. + assertTrue(selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(queueConsumer("receiver", 1, 10_000, 100), activeSource), true).isEmpty()); + } + + @Test + void testIneligibleReceiversAndDonors() { + final ProcessorDemand donor = queueConsumer("donor", 6, 1, 100); + final List ineligibleReceivers = List.of( + demand("not-ready", 1, false, false, false, 10_000, new RecentDemand(100, 10, 0, MEASURED_NANOS)), + demand("at-maximum", 12, true, false, false, 10_000, new RecentDemand(100, 10, 0, MEASURED_NANOS)), + demand("failing", 1, true, false, false, 10_000, new RecentDemand(100, 10, 1, MEASURED_NANOS)), + demand("comparing", 1, true, true, false, 10_000, new RecentDemand(100, 10, 0, MEASURED_NANOS)), + demand("cooling-down", 1, true, false, true, 10_000, new RecentDemand(100, 10, 0, MEASURED_NANOS)), + demand("newly-started", 1, true, false, false, 10_000, new RecentDemand(100, 10, 0, TimeUnit.SECONDS.toNanos(5))), + demand("not-progressing", 1, true, false, false, 10_000, new RecentDemand(0, 10, 0, MEASURED_NANOS))); + for (final ProcessorDemand ineligibleReceiver : ineligibleReceivers) { + assertTrue(selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(ineligibleReceiver, donor), true).isEmpty(), ineligibleReceiver.identifier()); + } + + // Failing and non-progressing Processors score 0, so they give up extra tasks first. + final ProcessorDemand receiver = queueConsumer("receiver", 1, 10_000, 100); + final ProcessorDemand busyDonor = queueConsumer("busy-donor", 6, 50, 100); + final ProcessorDemand failingDonor = demand("failing-donor", 6, true, false, false, 10_000, new RecentDemand(100, 10, 1, MEASURED_NANOS)); + final ProcessorDemand stalledDonor = demand("stalled-donor", 6, true, false, false, 10_000, new RecentDemand(0, 10, 0, MEASURED_NANOS)); + assertEquals(Optional.of(new TaskMove("receiver", "failing-donor")), selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(receiver, busyDonor, failingDonor), true)); + assertEquals(Optional.of(new TaskMove("receiver", "stalled-donor")), selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(receiver, busyDonor, stalledDonor), true)); + + // A newly started Processor with no commits yet has the maximum score, so it does not give up tasks. + assertEquals(ConcurrentTaskMoveSelector.MAX_SCORE, + ConcurrentTaskMoveSelector.calculateScore(demand("new", 6, true, false, false, 10_000, new RecentDemand(0, 0, 0, TimeUnit.SECONDS.toNanos(2))))); + // A donor keeps at least one task, and a donor measuring its own comparison is left alone. + final ProcessorDemand comparingDonor = demand("comparing", 6, true, true, false, 1, new RecentDemand(100, 10, 0, MEASURED_NANOS)); + assertTrue(selectSustainedMove(new ConcurrentTaskMoveSelector(), List.of(receiver, queueConsumer("single-task", 1, 1, 100), comparingDonor), true).isEmpty()); + } + + private static Optional selectSustainedMove(final ConcurrentTaskMoveSelector selector, final List demands, final boolean globalCapacityFull) { + Optional move = Optional.empty(); + for (int evaluation = 0; evaluation < ConcurrentTaskMoveSelector.SUSTAINED_EVALUATIONS; evaluation++) { + move = selector.selectMove(demands, globalCapacityFull); + } + + return move; + } + + private static ProcessorDemand queueConsumer(final String identifier, final int concurrentTasks, final long queuedFlowFiles, final double committedPerSecond) { + return demand(identifier, concurrentTasks, true, false, false, queuedFlowFiles, new RecentDemand(committedPerSecond, 10, 0, MEASURED_NANOS)); + } + + private static ProcessorDemand source(final boolean ready, final boolean recentlyActive, final double concurrentTaskUtilization) { + final ProcessorSchedulingSnapshot snapshot = ProcessorSchedulingSnapshot.createBuilder() + .setAppliedSettings(new SchedulingSettings(1, 0L)) + .setProcessorReady(ready) + .setSourceProcessorRecentlyReportedActivity(recentlyActive) + .setConcurrentTaskUtilization(concurrentTaskUtilization) + .setRecentDemand(new RecentDemand(0, 10, 0, MEASURED_NANOS)) + .build(); + return new ProcessorDemand("source", true, 6, 12, false, false, snapshot); + } + + private static ProcessorDemand demand(final String identifier, final int concurrentTasks, final boolean ready, final boolean comparisonActive, + final boolean coolingDown, final long queuedFlowFiles, final RecentDemand recentDemand) { + final ProcessorSchedulingSnapshot snapshot = ProcessorSchedulingSnapshot.createBuilder() + .setAppliedSettings(new SchedulingSettings(1, 0L)) + .setProcessorReady(ready) + .setConcurrentTaskUtilization(1D) + .setLocalInputQueueCount(queuedFlowFiles) + .setRecentDemand(recentDemand) + .build(); + return new ProcessorDemand(identifier, false, concurrentTasks, 12, comparisonActive, coolingDown, snapshot); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurementsTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurementsTest.java index 9a79a0d3a3c8..babc040723b5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurementsTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/ProcessorSchedulingMeasurementsTest.java @@ -55,8 +55,25 @@ void testOnlyCommittedWorkContributesToThroughput() { assertEquals(1, failed.completedInvocations()); assertEquals(1, failed.failedInvocations()); observer.onCommit(new CommittedSchedulingWork(2, 10)); - assertEquals(2, captureSnapshot(measurements, 2).committedFlowFiles()); + final ProcessorSchedulingSnapshot committed = captureSnapshot(measurements, 2); + assertEquals(2, committed.committedFlowFiles()); + // Until ten seconds are measured, the recent rates are the average of every one-second capture so far. + assertEquals(new ProcessorSchedulingSnapshot.RecentDemand(1D, 0.5D, 0.5D, TimeUnit.SECONDS.toNanos(2)), committed.recentDemand()); assertEquals(0, captureSnapshot(measurements, 3).committedFlowFiles()); + + ProcessorSchedulingSnapshot later = null; + for (int second = 0; second < 7; second++) { + later = captureSnapshot(measurements, 3); + } + + assertEquals(0.2D, later.recentDemand().committedInputFlowFilesPerSecond(), 0.000001D); + + // After ten seconds, each capture is weighted toward the most recent measurements, so older commits fade away. + for (int second = 0; second < 10; second++) { + later = captureSnapshot(measurements, 3); + } + + assertTrue(later.recentDemand().committedInputFlowFilesPerSecond() < 0.1D); } @Test @@ -64,24 +81,24 @@ void testLongInvocationsContributeUtilizationBeforeCompleting() { final AtomicLong nowNanos = new AtomicLong(); final ProcessorSchedulingMeasurements measurements = new ProcessorSchedulingMeasurements(nowNanos::get); final SchedulingSettings settings = new SchedulingSettings(2, 0L); - measurements.recordInvocationStarted(); + measurements.recordTaskBusy(); - // The scheduling agent reads its timestamp at 1000 milliseconds, then a second invocation starts at 1200 milliseconds before the snapshot is captured. + // The scheduling agent reads its timestamp at 1000 milliseconds, then a second task becomes busy at 1200 milliseconds before the snapshot is captured. final long agentTimestampNanos = TimeUnit.MILLISECONDS.toNanos(1000); nowNanos.set(TimeUnit.MILLISECONDS.toNanos(1200)); - measurements.recordInvocationStarted(); + measurements.recordTaskBusy(); final ProcessorSchedulingSnapshot first = measurements.captureSnapshot(agentTimestampNanos, agentTimestampNanos, settings, - true, true, 0D, false, false); - // 1200 milliseconds of invocation time out of 1200 milliseconds available to each of 2 tasks + true, true, 0L, 0D, false, false, false); + // 1200 milliseconds of busy time out of 1200 milliseconds available to each of 2 tasks assertEquals(0.5D, first.concurrentTaskUtilization(), 0.000001D); assertEquals(0, first.completedInvocations()); nowNanos.set(TimeUnit.MILLISECONDS.toNanos(2200)); - measurements.recordInvocationFinished(); + measurements.recordTaskIdle(); nowNanos.set(TimeUnit.MILLISECONDS.toNanos(3200)); final ProcessorSchedulingSnapshot second = measurements.captureSnapshot(nowNanos.get(), nowNanos.get() - agentTimestampNanos, settings, - true, true, 0D, false, false); - // 2000 milliseconds from the first invocation plus 1000 milliseconds from the second, out of 2000 milliseconds available to each of 2 tasks + true, true, 0L, 0D, false, false, false); + // 2000 milliseconds from the first task plus 1000 milliseconds from the second, out of 2000 milliseconds available to each of 2 tasks assertEquals(0.75D, second.concurrentTaskUtilization(), 0.000001D); } @@ -97,6 +114,6 @@ void testBatchActivityDistinguishesEachTrigger() { private ProcessorSchedulingSnapshot captureSnapshot(final ProcessorSchedulingMeasurements measurements, final int concurrency) { return measurements.captureSnapshot(System.nanoTime(), TimeUnit.SECONDS.toNanos(1), - new SchedulingSettings(concurrency, 0L), true, true, 0D, false, false); + new SchedulingSettings(concurrency, 0L), true, true, 0L, 0D, false, false, false); } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingControllerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingControllerTest.java index f0d4cdf59dce..ecaa8c9954f9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingControllerTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/auto/StandardAutoSchedulingControllerTest.java @@ -268,6 +268,43 @@ void testSharedBudgetAndFailuresBoundGrowth() { assertEquals(1, simulation.settings.concurrentTasks()); simulation.observe(1000); assertEquals(ProcessorSchedulingDecisionReason.PROCESSOR_INVOCATION_FAILED, simulation.lastDecision.reason()); + + // With a larger global limit that is fully used, increases stay blocked after the failures stop and the increase cooldown ends. + simulation.failedInvocations = 0; + simulation.systemMeasurements = SystemSchedulingSnapshot.createBuilder() + .setCpuLoadAvailable(true) + .setCpuLoad(0.25D) + .setAverageCpuLoad(0.25D) + .setMaxGlobalConcurrentTasks(32) + .setFractionOfSamplesWithAllGlobalTaskSlotsUsed(1D) + .setFractionOfGlobalTaskTimeSpentWaiting(0.5D) + .build(); + simulation.observe(15, 1000); + assertEquals(1, simulation.settings.concurrentTasks()); + assertEquals(ConcurrencyUpdateReason.GLOBAL_CAPACITY_FULL, simulation.controller.getConcurrencyUpdateStatus().reason()); + + // A single evaluation that is allowed to test despite full global capacity starts the normal one-task test, which is kept because + // throughput increases. + simulation.globalCapacityTestAllowed = true; + simulation.observe(1000); + simulation.globalCapacityTestAllowed = false; + assertEquals(ProcessorSchedulingDecisionReason.TESTING_HIGHER_CONCURRENCY, simulation.lastDecision.reason()); + assertEquals(2, simulation.settings.concurrentTasks()); + simulation.observe(6, 2000); + assertEquals(ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_INCREASED_THROUGHPUT, simulation.lastDecision.reason()); + assertEquals(2, simulation.settings.concurrentTasks()); + + // Without the allowance, the next increase is blocked again. With it, a test that does not increase throughput is rejected. + simulation.observe(2000); + assertEquals(2, simulation.settings.concurrentTasks()); + assertEquals(ConcurrencyUpdateReason.GLOBAL_CAPACITY_FULL, simulation.controller.getConcurrencyUpdateStatus().reason()); + simulation.globalCapacityTestAllowed = true; + simulation.observe(2000); + simulation.globalCapacityTestAllowed = false; + assertEquals(3, simulation.settings.concurrentTasks()); + simulation.observe(6, 2000); + assertEquals(ProcessorSchedulingDecisionReason.HIGHER_CONCURRENCY_DID_NOT_INCREASE_THROUGHPUT, simulation.lastDecision.reason()); + assertEquals(2, simulation.settings.concurrentTasks()); } @Test @@ -379,6 +416,7 @@ private static class Simulation { private boolean processorReady = true; private boolean inputQueueHasFlowFiles = true; private boolean sourceProcessorRecentlyReportedActivity; + private boolean globalCapacityTestAllowed; private Simulation(final int maxConcurrentTasks, final boolean batchingSupported, final boolean processorTriggeredSerially) { this(maxConcurrentTasks, batchingSupported, processorTriggeredSerially, false, Runtime.getRuntime().availableProcessors()); @@ -418,6 +456,7 @@ private void observe(final long committedFlowFiles) { .setProcessorReady(processorReady) .setInputQueueHasFlowFiles(inputQueueHasFlowFiles) .setSourceProcessorRecentlyReportedActivity(sourceProcessorRecentlyReportedActivity) + .setGlobalCapacityTestAllowed(globalCapacityTestAllowed) .build(); lastDecision = controller.evaluate(processorMeasurements, systemMeasurements); if (lastDecision.applySettings()) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 666f318b3090..73c321384baa 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -4083,6 +4083,9 @@ public ProcessorDiagnosticsDTO createProcessorDiagnosticsDto(final ProcessorNode autoSchedulingDiagnosticsDto.setCollectingMeasurements(autoSchedulingDiagnostics.collectingMeasurements()); autoSchedulingDiagnosticsDto.setFlowFileMeasurementsAvailable(autoSchedulingDiagnostics.flowFileMeasurementsAvailable()); autoSchedulingDiagnosticsDto.setConcurrencyIncreaseExplanation(autoSchedulingDiagnostics.concurrencyIncreaseExplanation()); + autoSchedulingDiagnosticsDto.setLocalInputQueueCount(autoSchedulingDiagnostics.localInputQueueCount()); + autoSchedulingDiagnosticsDto.setDemandScore(autoSchedulingDiagnostics.demandScore()); + autoSchedulingDiagnosticsDto.setTaskMoveRole(autoSchedulingDiagnostics.taskMoveRole()); procDiagnostics.setAutoSchedulingDiagnostics(autoSchedulingDiagnosticsDto); } From f356ad850633d263d8f1ad1ce4d5f062e942958e Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Tue, 29 Sep 2026 15:56:07 -0400 Subject: [PATCH 3/4] NIFI-16365 Retain permits for short non-batching Processor invocations Co-authored-by: Cursor --- .../VirtualThreadSchedulingAgent.java | 47 +++++++++++++++++-- .../VirtualThreadSchedulingAgentTest.java | 40 ++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java index a426d434ea92..da2deb62ff65 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgent.java @@ -79,6 +79,9 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.LockSupport; +import java.util.function.BooleanSupplier; +import java.util.function.LongSupplier; +import java.util.function.Supplier; /** * Scheduling agent that runs components on virtual threads. A {@link DynamicSemaphore} @@ -89,6 +92,7 @@ public class VirtualThreadSchedulingAgent implements SchedulingAgent { private static final Logger logger = LoggerFactory.getLogger(VirtualThreadSchedulingAgent.class); private static final long PERMIT_POLL_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(1L); + private static final long NON_BATCHED_PROCESSOR_BURST_NANOS = TimeUnit.MILLISECONDS.toNanos(10L); private final FlowController flowController; private final RepositoryContextFactory contextFactory; @@ -317,11 +321,17 @@ private void runAutoSchedulingLoop(final ProcessorAutoSchedulingState state, fin } invocationAttempted = true; - final InvocationObserver observer = state.measurements.createInvocationObserver(); - final InvocationResult result = state.connectableTask.invoke(schedulingSettings.runDurationNanos(), - () -> isActive(state.lifecycleState, state.generation) && !worker.retired.get() - && state.schedulingSettings.get() == schedulingSettings, observer); - state.recordInvocationResult(result); + final long burstNanos = state.batchingSupported || state.sourceProcessor ? 0L : NON_BATCHED_PROCESSOR_BURST_NANOS; + invokeWithBurst(burstNanos, System::nanoTime, + () -> shouldContinueNonBatchedProcessorBurst(state, worker, schedulingSettings), + () -> { + final InvocationObserver observer = state.measurements.createInvocationObserver(); + final InvocationResult result = state.connectableTask.invoke(schedulingSettings.runDurationNanos(), + () -> isActive(state.lifecycleState, state.generation) && !worker.retired.get() + && state.schedulingSettings.get() == schedulingSettings, observer); + state.recordInvocationResult(result); + return result; + }); } else { concurrentTaskSlotUnavailable = true; } @@ -354,6 +364,33 @@ private void runAutoSchedulingLoop(final ProcessorAutoSchedulingState state, fin } } + /** + * Allows a queue-consuming Processor without session batching support to perform additional independent invocations before + * releasing its global permit. Each invocation uses its normal Process Session and commit behavior. The bounded burst avoids + * making very short invocations repeatedly wait behind long-running invocations while preserving time-based fairness. The next + * invocation performs the authoritative input and backpressure readiness check, avoiding duplicate connection scans here. + */ + private boolean shouldContinueNonBatchedProcessorBurst(final ProcessorAutoSchedulingState state, final ProcessorTaskWorker worker, + final SchedulingSettings schedulingSettings) { + return !Thread.currentThread().isInterrupted() + && isActive(state.lifecycleState, state.generation) + && !worker.retired.get() + && state.schedulingSettings.get() == schedulingSettings; + } + + static InvocationResult invokeWithBurst(final long burstNanos, final LongSupplier nanoTimeSupplier, + final BooleanSupplier continueBurst, final Supplier invocation) { + final long burstDeadlineNanos = nanoTimeSupplier.getAsLong() + burstNanos; + InvocationResult result; + do { + result = invocation.get(); + } while (burstNanos > 0L + && result.getOutcome() == InvocationOutcome.INVOKED_WITH_ACTIVITY + && nanoTimeSupplier.getAsLong() < burstDeadlineNanos + && continueBurst.getAsBoolean()); + return result; + } + private void waitForAutoReadiness(final ProcessorAutoSchedulingState state, final ProcessorTaskWorker worker) { final long changeSequence = state.generation.getChangeSequence(); if (state.connectableTask.isReady() && state.isSourceWorkCheckAllowed()) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgentTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgentTest.java index 2bae49d829de..76780634689f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgentTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/VirtualThreadSchedulingAgentTest.java @@ -35,6 +35,8 @@ import org.apache.nifi.controller.scheduling.auto.SystemSchedulingMetrics; import org.apache.nifi.controller.scheduling.auto.SystemSchedulingSnapshot; import org.apache.nifi.controller.status.FlowFileAvailability; +import org.apache.nifi.controller.tasks.InvocationOutcome; +import org.apache.nifi.controller.tasks.InvocationResult; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarThreadContextClassLoader; @@ -161,6 +163,44 @@ void testProcessContextConcurrencyLimit() { assertEquals(1, agent.getProcessContextConcurrencyLimit(automatic)); } + @Test + void testInvocationBurstContinuesWhileActivityAndTimeRemain() { + final AtomicInteger invocationCount = new AtomicInteger(); + final AtomicInteger continuationCount = new AtomicInteger(); + final AtomicLong nanoTime = new AtomicLong(); + + final InvocationResult result = VirtualThreadSchedulingAgent.invokeWithBurst(10L, () -> nanoTime.getAndAdd(4L), + () -> { + continuationCount.incrementAndGet(); + return true; + }, + () -> { + invocationCount.incrementAndGet(); + return InvocationResult.completed(InvocationOutcome.INVOKED_WITH_ACTIVITY); + }); + + assertEquals(InvocationOutcome.INVOKED_WITH_ACTIVITY, result.getOutcome()); + assertEquals(3, invocationCount.get()); + assertEquals(2, continuationCount.get()); + } + + @Test + void testInvocationBurstStopsWithoutActivityOrContinuation() { + final AtomicInteger noActivityInvocations = new AtomicInteger(); + VirtualThreadSchedulingAgent.invokeWithBurst(10L, () -> 0L, () -> true, () -> { + noActivityInvocations.incrementAndGet(); + return InvocationResult.completed(InvocationOutcome.INVOKED_WITHOUT_ACTIVITY); + }); + assertEquals(1, noActivityInvocations.get()); + + final AtomicInteger stoppedInvocations = new AtomicInteger(); + VirtualThreadSchedulingAgent.invokeWithBurst(10L, () -> 0L, () -> false, () -> { + stoppedInvocations.incrementAndGet(); + return InvocationResult.completed(InvocationOutcome.INVOKED_WITH_ACTIVITY); + }); + assertEquals(1, stoppedInvocations.get()); + } + @Test void testAutomaticSchedulingBacksOffEmptySource() throws InterruptedException { final AtomicInteger invocationCount = new AtomicInteger(); From e62937dbe31e695353466e08022fb9c9dcbf1eb1 Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Wed, 30 Sep 2026 11:21:11 -0400 Subject: [PATCH 4/4] NIFI-16365 Verify AUTO scheduling support in the REST tier only - Restored the no-op StandardNiFiServiceFacade.verifyComponentTypes so that flows with missing Processor types can be imported and pasted as ghost Processors - Removed AUTO scheduling checks from the flow synchronizer and flow snippet so that loading a flow on startup or cluster join never fails because of an unsupported scheduling strategy; validation marks such Processors invalid - Added the AUTO scheduling check to StandardProcessorDAO.verifyCreate, alongside the existing check in verifyUpdate - Removed AutoSchedulingIT.testContinuousFlowAdaptsToBlockingWork, which depended on the host having spare CPU Co-authored-by: Cursor --- .../nifi/controller/ProcessorDetails.java | 17 +--- ...tandardVersionedComponentSynchronizer.java | 18 ---- ...ardVersionedComponentSynchronizerTest.java | 36 ++++---- .../nifi/controller/FlowController.java | 6 -- .../nifi/controller/StandardFlowSnippet.java | 13 --- .../controller/StandardFlowSnippetTest.java | 87 ------------------- .../nifi/web/StandardNiFiServiceFacade.java | 1 - .../web/dao/impl/StandardProcessorDAO.java | 15 +++- .../web/StandardNiFiServiceFacadeTest.java | 11 --- .../dao/impl/StandardProcessorDAOTest.java | 28 +++++- .../nifi/tests/system/pg/CopyPasteIT.java | 6 +- .../system/scheduling/AutoSchedulingIT.java | 37 -------- 12 files changed, 65 insertions(+), 210 deletions(-) delete mode 100644 nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowSnippetTest.java diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/ProcessorDetails.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/ProcessorDetails.java index 711c9df45004..76d6e381da98 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/ProcessorDetails.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/ProcessorDetails.java @@ -54,8 +54,7 @@ public ProcessorDetails(final LoggableComponent processor) { this.triggerWhenEmpty = procClass.isAnnotationPresent(TriggerWhenEmpty.class); this.sideEffectFree = procClass.isAnnotationPresent(SideEffectFree.class); this.batchSupported = procClass.isAnnotationPresent(SupportsBatching.class); - final AllowsAutoScheduling allowsAutoScheduling = procClass.getAnnotation(AllowsAutoScheduling.class); - this.autoSchedulingSupported = allowsAutoScheduling == null || allowsAutoScheduling.value(); + this.autoSchedulingSupported = isAutoSchedulingSupported(procClass); this.triggeredSerially = procClass.isAnnotationPresent(TriggerSerially.class); this.triggerWhenAnyDestinationAvailable = procClass.isAnnotationPresent(TriggerWhenAnyDestinationAvailable.class); this.executionNodeRestricted = procClass.isAnnotationPresent(PrimaryNodeOnly.class); @@ -100,17 +99,9 @@ public boolean isAutoSchedulingSupported() { return autoSchedulingSupported; } - public static void verifyAutoSchedulingSupported(final Object component, final String name, final String identifier) { - if (!(component instanceof final Processor processor)) { - throw new IllegalStateException("Processor " + name + " [" + identifier - + "] cannot use scheduling strategy AUTO because its automatic scheduling capability could not be resolved"); - } - - final AllowsAutoScheduling capability = processor.getClass().getAnnotation(AllowsAutoScheduling.class); - if (capability != null && !capability.value()) { - throw new IllegalStateException("Processor " + name + " [" + identifier - + "] cannot use scheduling strategy AUTO because its implementation disables automatic scheduling"); - } + public static boolean isAutoSchedulingSupported(final Class processorClass) { + final AllowsAutoScheduling allowsAutoScheduling = processorClass.getAnnotation(AllowsAutoScheduling.class); + return allowsAutoScheduling == null || allowsAutoScheduling.value(); } public boolean isExecutionNodeRestricted() { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java index bc6debcac123..e7930d84c329 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java @@ -37,7 +37,6 @@ import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; -import org.apache.nifi.controller.ProcessorDetails; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.PropertyConfiguration; import org.apache.nifi.controller.ReportingTaskNode; @@ -2822,10 +2821,6 @@ private void updateLabel(final Label label, final VersionedLabel proposed) { private ProcessorNode addProcessor(final ProcessGroup destination, final VersionedProcessor proposed, final ComponentIdGenerator componentIdGenerator, final ProcessGroup topLevelGroup) throws ProcessorInstantiationException { - if (SchedulingStrategy.AUTO.name().equals(proposed.getSchedulingStrategy())) { - verifyAutoSchedulingSupported(proposed); - } - final String identifier = componentIdGenerator.generateUuid(proposed.getIdentifier(), proposed.getInstanceIdentifier(), destination.getIdentifier()); LOG.debug("Adding Processor with ID {} of type {}", identifier, proposed.getType()); @@ -2904,10 +2899,6 @@ public void synchronize(final ProcessorNode processor, final VersionedProcessor return; } - if (proposedProcessor != null && SchedulingStrategy.AUTO.name().equals(proposedProcessor.getSchedulingStrategy())) { - verifyAutoSchedulingSupported(proposedProcessor); - } - setSynchronizationOptions(synchronizationOptions); final long timeout = System.currentTimeMillis() + synchronizationOptions.getComponentStopTimeout().toMillis(); @@ -3152,9 +3143,6 @@ private void stopControllerService(final ControllerServiceNode controllerService private void updateProcessor(final ProcessorNode processor, final VersionedProcessor proposed, final ProcessGroup topLevelGroup) throws ProcessorInstantiationException { LOG.debug("Updating Processor {}", processor); - if (SchedulingStrategy.AUTO.name().equals(proposed.getSchedulingStrategy())) { - verifyAutoSchedulingSupported(proposed); - } processor.pauseValidationTrigger(); try { @@ -3214,12 +3202,6 @@ private void updateProcessor(final ProcessorNode processor, final VersionedProce } } - private void verifyAutoSchedulingSupported(final VersionedProcessor processor) { - final BundleCoordinate coordinate = toCoordinate(processor.getBundle()); - final Object temporaryComponent = context.getExtensionManager().getTempComponent(processor.getType(), coordinate); - ProcessorDetails.verifyAutoSchedulingSupported(temporaryComponent, processor.getName(), processor.getIdentifier()); - } - private String getServiceInstanceId(final String serviceVersionedComponentId, final ProcessGroup group) { for (final ControllerServiceNode serviceNode : group.getControllerServices(false)) { final String versionedId = serviceNode.getVersionedComponentId().orElse( diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java index 79652b34bb78..3f6be1df32a1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizerTest.java @@ -591,24 +591,24 @@ public void testSynchronizeStoppedProcessor() throws FlowSynchronizationExceptio verify(componentScheduler, times(0)).startComponent(any(Connectable.class)); } + /** + * Synchronization is used to load the flow on startup and when joining a cluster, so a Processor configured with a scheduling + * strategy that its implementation does not support must still be created and updated. Validation marks such a Processor invalid. + */ @Test - void testSynchronizeRejectsUnresolvedAutomaticProcessorCreateBeforeMutation() { - final VersionedProcessor versionedProcessor = createUnsupportedAutomaticVersionedProcessor(); - - assertThrows(IllegalStateException.class, () -> synchronizer.synchronize(null, versionedProcessor, group, synchronizationOptions)); - verify(componentScheduler, never()).pause(); - verify(flowManager, never()).createProcessor(anyString(), anyString(), any(BundleCoordinate.class), anyBoolean()); - } - - @Test - void testSynchronizeRejectsUnsupportedAutomaticProcessorUpdateBeforeMutation() { - final VersionedProcessor versionedProcessor = createUnsupportedAutomaticVersionedProcessor(); + void testSynchronizeAppliesAutoSchedulingToProcessorWithoutAutoScheduling() throws FlowSynchronizationException, TimeoutException, InterruptedException { + final VersionedProcessor versionedProcessor = createVersionedProcessorWithoutAutoScheduling(); final BundleCoordinate coordinate = new BundleCoordinate(bundle.getGroup(), bundle.getArtifact(), bundle.getVersion()); - when(extensionManager.getTempComponent(versionedProcessor.getType(), coordinate)).thenReturn(mock(UnsupportedAutomaticProcessor.class)); + when(extensionManager.getTempComponent(versionedProcessor.getType(), coordinate)).thenReturn(mock(ProcessorWithoutAutoScheduling.class)); + final ProcessorNode processorNode = createMockProcessor(); + when(flowManager.createProcessor(any(), any(), any(), eq(true))).thenReturn(processorNode); - assertThrows(IllegalStateException.class, () -> synchronizer.synchronize(processorA, versionedProcessor, group, synchronizationOptions)); - verify(componentScheduler, never()).pause(); - verify(processorA, never()).setSchedulingStrategy(any(SchedulingStrategy.class)); + synchronizer.synchronize(null, versionedProcessor, group, synchronizationOptions); + verify(flowManager).createProcessor(eq(versionedProcessor.getType()), anyString(), any(BundleCoordinate.class), eq(true)); + verify(processorNode).setSchedulingStrategy(SchedulingStrategy.AUTO); + + synchronizer.synchronize(processorA, versionedProcessor, group, synchronizationOptions); + verify(processorA).setSchedulingStrategy(SchedulingStrategy.AUTO); } @Test @@ -2123,9 +2123,9 @@ private VersionedProcessor createMinimalVersionedProcessor() { return versionedProcessor; } - private VersionedProcessor createUnsupportedAutomaticVersionedProcessor() { + private VersionedProcessor createVersionedProcessorWithoutAutoScheduling() { final VersionedProcessor versionedProcessor = createMinimalVersionedProcessor(); - versionedProcessor.setType(UnsupportedAutomaticProcessor.class.getName()); + versionedProcessor.setType(ProcessorWithoutAutoScheduling.class.getName()); versionedProcessor.setSchedulingStrategy(SchedulingStrategy.AUTO.name()); return versionedProcessor; } @@ -2222,7 +2222,7 @@ private record ScheduledStateUpdate(T component, org.apache.nifi.controller.S } @AllowsAutoScheduling(false) - private abstract static class UnsupportedAutomaticProcessor implements Processor { + private abstract static class ProcessorWithoutAutoScheduling implements Processor { } private record ControllerServiceStateUpdate(ControllerServiceNode controllerService, ControllerServiceState state) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 408a3032e40c..73b8d5561c5b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -2394,12 +2394,6 @@ private void verifyProcessorsInVersionedFlow(final VersionedProcessGroup version if (supportedTypes.containsKey(processor.getType())) { verifyBundleInVersionedFlow(processor.getBundle(), supportedTypes.get(processor.getType())); - if (SchedulingStrategy.AUTO.name().equals(processor.getSchedulingStrategy())) { - final Bundle bundle = processor.getBundle(); - final BundleCoordinate coordinate = new BundleCoordinate(bundle.getGroup(), bundle.getArtifact(), bundle.getVersion()); - final Object temporaryComponent = extensionManager.getTempComponent(processor.getType(), coordinate); - ProcessorDetails.verifyAutoSchedulingSupported(temporaryComponent, processor.getName(), processor.getIdentifier()); - } } else { throw new IllegalStateException("Invalid Processor Type: " + processor.getType()); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java index f286c1d37edc..a495adb1fcde 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java @@ -209,7 +209,6 @@ private void verifyProcessorsInSnippet(final FlowSnippetDTO templateContents, fi if (supportedTypes.containsKey(processor.getType())) { verifyBundleInSnippet(processor.getBundle(), supportedTypes.get(processor.getType())); - verifyAutoSchedulingSupported(processor); } else { throw new IllegalStateException("Invalid Processor Type: " + processor.getType()); } @@ -221,18 +220,6 @@ private void verifyProcessorsInSnippet(final FlowSnippetDTO templateContents, fi } } - private void verifyAutoSchedulingSupported(final ProcessorDTO processor) { - final ProcessorConfigDTO config = processor.getConfig(); - if (config == null || !SchedulingStrategy.AUTO.name().equals(config.getSchedulingStrategy())) { - return; - } - - final BundleDTO bundle = processor.getBundle(); - final BundleCoordinate coordinate = new BundleCoordinate(bundle.getGroup(), bundle.getArtifact(), bundle.getVersion()); - final Object temporaryComponent = extensionManager.getTempComponent(processor.getType(), coordinate); - ProcessorDetails.verifyAutoSchedulingSupported(temporaryComponent, processor.getName(), processor.getId()); - } - public void instantiate(final FlowManager flowManager, final FlowController flowController, final ProcessGroup group, final boolean topLevel) { // // Instantiate Controller Services diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowSnippetTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowSnippetTest.java deleted file mode 100644 index 3a8cc6741cce..000000000000 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowSnippetTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller; - -import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; -import org.apache.nifi.bundle.Bundle; -import org.apache.nifi.bundle.BundleCoordinate; -import org.apache.nifi.bundle.BundleDetails; -import org.apache.nifi.nar.ExtensionDefinition; -import org.apache.nifi.nar.ExtensionManager; -import org.apache.nifi.processor.Processor; -import org.apache.nifi.scheduling.SchedulingStrategy; -import org.apache.nifi.web.api.dto.BundleDTO; -import org.apache.nifi.web.api.dto.FlowSnippetDTO; -import org.apache.nifi.web.api.dto.ProcessorConfigDTO; -import org.apache.nifi.web.api.dto.ProcessorDTO; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -class StandardFlowSnippetTest { - - @Test - void testUnsupportedAutomaticProcessorRejectedBeforeInstantiation() { - final BundleCoordinate coordinate = new BundleCoordinate("group", "artifact", "version"); - final BundleDTO bundle = new BundleDTO(); - bundle.setGroup(coordinate.getGroup()); - bundle.setArtifact(coordinate.getId()); - bundle.setVersion(coordinate.getVersion()); - - final ProcessorConfigDTO config = new ProcessorConfigDTO(); - config.setSchedulingStrategy(SchedulingStrategy.AUTO.name()); - final ProcessorDTO processor = new ProcessorDTO(); - processor.setId("processor-id"); - processor.setName("Unsupported Processor"); - processor.setType(UnsupportedAutomaticProcessor.class.getName()); - processor.setBundle(bundle); - processor.setConfig(config); - final FlowSnippetDTO snippet = new FlowSnippetDTO(); - snippet.setProcessors(Set.of(processor)); - - final ExtensionDefinition definition = mock(ExtensionDefinition.class); - when(definition.getImplementationClassName()).thenReturn(processor.getType()); - final BundleDetails bundleDetails = mock(BundleDetails.class); - when(bundleDetails.getCoordinate()).thenReturn(coordinate); - final Bundle installedBundle = mock(Bundle.class); - when(installedBundle.getBundleDetails()).thenReturn(bundleDetails); - final ExtensionManager extensionManager = mock(ExtensionManager.class); - when(extensionManager.getExtensions(Processor.class)).thenReturn(Set.of(definition)); - when(extensionManager.getBundles(processor.getType())).thenReturn(List.of(installedBundle)); - - final StandardFlowSnippet flowSnippet = new StandardFlowSnippet(snippet, extensionManager); - final IllegalStateException unresolvedException = assertThrows(IllegalStateException.class, flowSnippet::verifyComponentTypesInSnippet); - assertEquals("Processor Unsupported Processor [processor-id] cannot use scheduling strategy AUTO because its automatic scheduling capability could not be resolved", - unresolvedException.getMessage()); - - when(extensionManager.getTempComponent(processor.getType(), coordinate)).thenReturn(mock(UnsupportedAutomaticProcessor.class)); - final IllegalStateException exception = assertThrows(IllegalStateException.class, flowSnippet::verifyComponentTypesInSnippet); - - assertEquals("Processor Unsupported Processor [processor-id] cannot use scheduling strategy AUTO because its implementation disables automatic scheduling", - exception.getMessage()); - } - - @AllowsAutoScheduling(false) - private abstract static class UnsupportedAutomaticProcessor implements Processor { - } -} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index e818f506de0e..ad76675bf187 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -3087,7 +3087,6 @@ public boolean isRemoteGroupPortConnected(final String remoteProcessGroupId, fin @Override public void verifyComponentTypes(final VersionedProcessGroup versionedGroup) { - controllerFacade.verifyComponentTypes(versionedGroup); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java index c6278753fdcd..9191a0cb33bb 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardProcessorDAO.java @@ -29,6 +29,7 @@ import org.apache.nifi.connectable.Position; import org.apache.nifi.controller.BackoffMechanism; import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorDetails; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.controller.exception.ComponentLifeCycleException; @@ -123,7 +124,19 @@ public boolean hasProcessor(String id) { @Override public void verifyCreate(final ProcessorDTO processorDTO) { - verifyCreate(flowController.getExtensionManager(), processorDTO.getType(), processorDTO.getBundle()); + final ExtensionManager extensionManager = flowController.getExtensionManager(); + verifyCreate(extensionManager, processorDTO.getType(), processorDTO.getBundle()); + + final ProcessorConfigDTO config = processorDTO.getConfig(); + if (config == null || !SchedulingStrategy.AUTO.name().equals(config.getSchedulingStrategy())) { + return; + } + + final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(extensionManager, processorDTO.getType(), processorDTO.getBundle()); + final ConfigurableComponent temporaryComponent = extensionManager.getTempComponent(processorDTO.getType(), bundleCoordinate); + if (temporaryComponent != null && !ProcessorDetails.isAutoSchedulingSupported(temporaryComponent.getClass())) { + throw new ValidationException(List.of("Scheduling strategy AUTO is not supported by Processor type " + processorDTO.getType())); + } } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java index 87c39730b27b..47f729e17630 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java @@ -405,17 +405,6 @@ public Resource getResource() { } - @Test - public void testVerifyComponentTypesDelegatesToControllerFacade() { - final ControllerFacade controllerFacade = mock(ControllerFacade.class); - final VersionedProcessGroup versionedGroup = new VersionedProcessGroup(); - serviceFacade.setControllerFacade(controllerFacade); - - serviceFacade.verifyComponentTypes(versionedGroup); - - verify(controllerFacade).verifyComponentTypes(versionedGroup); - } - @Test public void testGetComponentsAffectedByFlowUpdate_WithNewStatelessProcessGroup_ReproducesNPE() { final String groupId = UUID.randomUUID().toString(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java index d0317e31c904..b6487d14e5e5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/StandardProcessorDAOTest.java @@ -16,6 +16,7 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.annotation.behavior.AllowsAutoScheduling; import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.bundle.BundleDetails; @@ -61,6 +62,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -211,7 +213,7 @@ void testVerifyUpdate_ProcessorNotFound() { } @Test - void testVerifyUpdateRejectsUnsupportedAutomaticScheduling() { + void testVerifyRejectsAutoSchedulingForProcessorWithoutAutoScheduling(@TempDir final File tempDir) { final ProcessorConfigDTO config = new ProcessorConfigDTO(); config.setSchedulingStrategy(SchedulingStrategy.AUTO.name()); final ProcessorDTO processorDTO = new ProcessorDTO(); @@ -220,10 +222,24 @@ void testVerifyUpdateRejectsUnsupportedAutomaticScheduling() { when(processorNode.isAutoSchedulingSupported()).thenReturn(false); when(processorNode.getName()).thenReturn("Unsupported Processor"); - final ValidationException exception = assertThrows(ValidationException.class, () -> dao.verifyUpdate(processorDTO)); + final ValidationException updateException = assertThrows(ValidationException.class, () -> dao.verifyUpdate(processorDTO)); + assertEquals(1, updateException.getValidationErrors().size()); + assertTrue(updateException.getValidationErrors().getFirst().contains("AUTO")); - assertEquals(List.of("Scheduling strategy AUTO is not supported by Processor Unsupported Processor [test-processor-id]"), - exception.getValidationErrors()); + final String processorType = ProcessorWithoutAutoScheduling.class.getName(); + processorDTO.setType(processorType); + final BundleCoordinate bundleCoordinate = new BundleCoordinate(BUNDLE_GROUP_ID, processorType, BUNDLE_VERSION); + final BundleDetails bundleDetails = new BundleDetails.Builder().coordinate(bundleCoordinate).workingDir(tempDir).build(); + when(flowController.getExtensionManager()).thenReturn(extensionManager); + when(extensionManager.getBundles(processorType)).thenReturn(List.of(new Bundle(bundleDetails, getClass().getClassLoader()))); + when(extensionManager.getTempComponent(processorType, bundleCoordinate)).thenReturn(mock(ProcessorWithoutAutoScheduling.class)); + + final ValidationException createException = assertThrows(ValidationException.class, () -> dao.verifyCreate(processorDTO)); + assertEquals(1, createException.getValidationErrors().size()); + assertTrue(createException.getValidationErrors().getFirst().contains("AUTO")); + + when(extensionManager.getTempComponent(processorType, bundleCoordinate)).thenReturn(processor); + dao.verifyCreate(processorDTO); } @Test @@ -283,4 +299,8 @@ void testCreateProcessor(@TempDir final File tempDir) { assertEquals(processorNode, createdProcessorNode); verify(processorNode).setProcessGroup(processGroup); } + + @AllowsAutoScheduling(false) + private abstract static class ProcessorWithoutAutoScheduling implements Processor { + } } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/pg/CopyPasteIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/pg/CopyPasteIT.java index 9adbcb2f52c0..2bd2b7b8adc3 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/pg/CopyPasteIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/pg/CopyPasteIT.java @@ -19,6 +19,7 @@ import org.apache.nifi.tests.system.NiFiSystemIT; import org.apache.nifi.toolkit.client.NiFiClientException; +import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.VersionControlInformationDTO; import org.apache.nifi.web.api.dto.flow.FlowDTO; import org.apache.nifi.web.api.entity.ControllerServiceEntity; @@ -164,6 +165,7 @@ public void testPastedPortInChildGroupHonorsCopiedPayload() throws NiFiClientExc public void testGhostComponent() throws NiFiClientException, IOException { final ProcessGroupEntity topLevel = getClientUtil().createProcessGroup("parent group", "root"); final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile", topLevel.getId()); + getClientUtil().updateProcessorSchedulingStrategy(generate, "AUTO"); final CopyRequestEntity copyRequestEntity = new CopyRequestEntity(); copyRequestEntity.setProcessors(Set.of(generate.getId())); @@ -181,7 +183,9 @@ public void testGhostComponent() throws NiFiClientException, IOException { final PasteResponseEntity pasteResponseEntity = getNifiClient().getProcessGroupClient().paste(topLevel.getId(), pasteRequestEntity); final FlowDTO flowDto = pasteResponseEntity.getFlow(); assertEquals(1, flowDto.getProcessors().size()); - assertTrue(flowDto.getProcessors().iterator().next().getComponent().getExtensionMissing()); + final ProcessorDTO ghostProcessor = flowDto.getProcessors().iterator().next().getComponent(); + assertTrue(ghostProcessor.getExtensionMissing()); + assertEquals("AUTO", ghostProcessor.getConfig().getSchedulingStrategy()); } @Test diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingIT.java index 84edfaa1291a..f5755ac24604 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/scheduling/AutoSchedulingIT.java @@ -29,7 +29,6 @@ import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; public class AutoSchedulingIT extends NiFiSystemIT { private static final int BACKPRESSURE_COUNT = 100; @@ -70,42 +69,6 @@ public void testSourceAndQueueConsumerWithAutomaticScheduling() throws NiFiClien assertEquals(0, getConnectionQueueSize(connection.getId())); } - @Test - @Timeout(value = 2, unit = TimeUnit.MINUTES) - public void testContinuousFlowAdaptsToBlockingWork() throws NiFiClientException, IOException, InterruptedException { - final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile"); - final ProcessorEntity sleep = getClientUtil().createProcessor("Sleep"); - final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); - final ConnectionEntity input = getClientUtil().createConnection(generate, sleep, "success"); - final ConnectionEntity output = getClientUtil().createConnection(sleep, terminate, "success"); - getClientUtil().updateConnectionBackpressure(input, BACKPRESSURE_COUNT, 10_000_000L); - getClientUtil().updateConnectionBackpressure(output, BACKPRESSURE_COUNT, 10_000_000L); - getClientUtil().updateProcessorProperties(sleep, Map.of("onTrigger Sleep Time", "100 ms")); - getClientUtil().updateProcessorSchedulingStrategy(generate, "AUTO"); - getClientUtil().updateProcessorSchedulingStrategy(sleep, "AUTO"); - getClientUtil().updateProcessorSchedulingStrategy(terminate, "AUTO"); - - getClientUtil().startProcessor(terminate); - getClientUtil().startProcessor(sleep); - getClientUtil().startProcessor(generate); - try { - final boolean adaptive = "VIRTUAL".equals(getNifiPropertiesOverrides().get("nifi.scheduling.strategy")); - final int expectedActiveTasks = adaptive ? 4 : 1; - waitFor(() -> getNifiClient().getProcessorClient().getProcessor(sleep.getId()).getStatus().getAggregateSnapshot().getActiveThreadCount() >= expectedActiveTasks); - waitFor(() -> getNifiClient().getProcessorClient().getProcessor(terminate.getId()).getStatus().getAggregateSnapshot().getFlowFilesIn() > 0); - assertCanonicalAutoConfiguration(getNifiClient().getProcessorClient().getProcessor(sleep.getId())); - assertTrue(getConnectionQueueSize(input.getId()) > 0); - - getClientUtil().stopProcessor(generate); - waitForQueueCount(input, 0); - waitForQueueCount(output, 0); - } finally { - getClientUtil().stopProcessor(generate); - getClientUtil().stopProcessor(sleep); - getClientUtil().stopProcessor(terminate); - } - } - private void assertCanonicalAutoConfiguration(final ProcessorEntity processorEntity) { final ProcessorConfigDTO config = processorEntity.getComponent().getConfig(); assertEquals("AUTO", config.getSchedulingStrategy());