Skip to content

Commit

Permalink
HDDS-9377. Add tests + minor code improvements.
Browse files Browse the repository at this point in the history
  • Loading branch information
@juncevich
juncevich committed Feb 10, 2025
1 parent 36c4043 commit 0ac69a3
Show file tree
Hide file tree
Showing 28 changed files with 1,618 additions and 413 deletions.
50 changes: 25 additions & 25 deletions hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ContainerClientMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.metrics.MutableQuantiles;
import org.apache.hadoop.ozone.metrics.OzoneMutableQuantiles;
import org.apache.hadoop.ozone.metrics.OzoneMutableRate;
import org.apache.hadoop.ozone.util.MetricUtil;
import org.apache.hadoop.ozone.metrics.OzoneMetricsSystem;
Expand Down Expand Up @@ -69,14 +69,14 @@ public final class ContainerClientMetrics {
private MutableCounterLong flushesDuringWrite;


private MutableQuantiles[] listBlockLatency;
private MutableQuantiles[] getBlockLatency;
private MutableQuantiles[] getCommittedBlockLengthLatency;
private MutableQuantiles[] readChunkLatency;
private MutableQuantiles[] getSmallFileLatency;
private MutableQuantiles[] hsyncLatencyNs;
private MutableQuantiles[] omHsyncLatencyNs;
private MutableQuantiles[] datanodeHsyncLatencyNs;
private OzoneMutableQuantiles[] listBlockLatency;
private OzoneMutableQuantiles[] getBlockLatency;
private OzoneMutableQuantiles[] getCommittedBlockLengthLatency;
private OzoneMutableQuantiles[] readChunkLatency;
private OzoneMutableQuantiles[] getSmallFileLatency;
private OzoneMutableQuantiles[] hsyncLatencyNs;
private OzoneMutableQuantiles[] omHsyncLatencyNs;
private OzoneMutableQuantiles[] datanodeHsyncLatencyNs;
private final Map<PipelineID, MutableCounterLong> writeChunkCallsByPipeline;
private final Map<PipelineID, MutableCounterLong> writeChunkBytesByPipeline;
private final Map<UUID, MutableCounterLong> writeChunksCallsByLeaders;
Expand Down Expand Up @@ -112,14 +112,14 @@ private ContainerClientMetrics() {
writeChunkBytesByPipeline = new ConcurrentHashMap<>();
writeChunksCallsByLeaders = new ConcurrentHashMap<>();

listBlockLatency = new MutableQuantiles[3];
getBlockLatency = new MutableQuantiles[3];
getCommittedBlockLengthLatency = new MutableQuantiles[3];
readChunkLatency = new MutableQuantiles[3];
getSmallFileLatency = new MutableQuantiles[3];
hsyncLatencyNs = new MutableQuantiles[3];
omHsyncLatencyNs = new MutableQuantiles[3];
datanodeHsyncLatencyNs = new MutableQuantiles[3];
listBlockLatency = new OzoneMutableQuantiles[3];
getBlockLatency = new OzoneMutableQuantiles[3];
getCommittedBlockLengthLatency = new OzoneMutableQuantiles[3];
readChunkLatency = new OzoneMutableQuantiles[3];
getSmallFileLatency = new OzoneMutableQuantiles[3];
hsyncLatencyNs = new OzoneMutableQuantiles[3];
omHsyncLatencyNs = new OzoneMutableQuantiles[3];
datanodeHsyncLatencyNs = new OzoneMutableQuantiles[3];
int[] intervals = {60, 300, 900};
for (int i = 0; i < intervals.length; i++) {
int interval = intervals[i];
Expand Down Expand Up @@ -196,63 +196,63 @@ public void recordWriteChunk(Pipeline pipeline, long chunkSizeBytes) {
}

public void addListBlockLatency(long latency) {
for (MutableQuantiles q : listBlockLatency) {
for (OzoneMutableQuantiles q : listBlockLatency) {
if (q != null) {
q.add(latency);
}
}
}

public void addHsyncLatency(long hsyncLatencyTime) {
for (MutableQuantiles q : hsyncLatencyNs) {
for (OzoneMutableQuantiles q : hsyncLatencyNs) {
if (q != null) {
q.add(hsyncLatencyTime);
}
}
}

public void addGetBlockLatency(long latency) {
for (MutableQuantiles q : getBlockLatency) {
for (OzoneMutableQuantiles q : getBlockLatency) {
if (q != null) {
q.add(latency);
}
}
}

public void addOMHsyncLatency(long hsyncLatencyTime) {
for (MutableQuantiles q : omHsyncLatencyNs) {
for (OzoneMutableQuantiles q : omHsyncLatencyNs) {
if (q != null) {
q.add(hsyncLatencyTime);
}
}
}

public void addGetCommittedBlockLengthLatency(long latency) {
for (MutableQuantiles q : getCommittedBlockLengthLatency) {
for (OzoneMutableQuantiles q : getCommittedBlockLengthLatency) {
if (q != null) {
q.add(latency);
}
}
}

public void addReadChunkLatency(long latency) {
for (MutableQuantiles q : readChunkLatency) {
for (OzoneMutableQuantiles q : readChunkLatency) {
if (q != null) {
q.add(latency);
}
}
}

public void addGetSmallFileLatency(long latency) {
for (MutableQuantiles q : getSmallFileLatency) {
for (OzoneMutableQuantiles q : getSmallFileLatency) {
if (q != null) {
q.add(latency);
}
}
}

public void addDataNodeHsyncLatency(long hsyncLatencyTime) {
for (MutableQuantiles q : datanodeHsyncLatencyNs) {
for (OzoneMutableQuantiles q : datanodeHsyncLatencyNs) {
if (q != null) {
q.add(hsyncLatencyTime);
}
Expand Down
14 changes: 7 additions & 7 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.metrics.MutableQuantiles;
import org.apache.hadoop.ozone.metrics.OzoneMutableQuantiles;
import org.apache.hadoop.ozone.util.MetricUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -67,9 +67,9 @@ public GrpcMetrics(Configuration conf) {
grpcQuantileEnable = (intervals.length > 0);
if (grpcQuantileEnable) {
grpcQueueTimeMillisQuantiles =
new MutableQuantiles[intervals.length];
new OzoneMutableQuantiles[intervals.length];
grpcProcessingTimeMillisQuantiles =
new MutableQuantiles[intervals.length];
new OzoneMutableQuantiles[intervals.length];
for (int i = 0; i < intervals.length; i++) {
int interval = intervals[i];
grpcQueueTimeMillisQuantiles[i] = OzoneMetricsSystem
Expand Down Expand Up @@ -129,14 +129,14 @@ public synchronized void getMetrics(MetricsCollector collector, boolean all) {

// There should be no getter method to avoid
// exposing internal representation. FindBugs error raised.
private MutableQuantiles[] grpcQueueTimeMillisQuantiles;
private OzoneMutableQuantiles[] grpcQueueTimeMillisQuantiles;

@Metric("Processsing time")
private OzoneMutableRate grpcProcessingTime;

// There should be no getter method to avoid
// exposing internal representation. FindBugs error raised.
private MutableQuantiles[] grpcProcessingTimeMillisQuantiles;
private OzoneMutableQuantiles[] grpcProcessingTimeMillisQuantiles;

@Metric("Number of active clients connected")
private MutableCounterLong numOpenClientConnections;
Expand All @@ -160,7 +160,7 @@ public void incrUnknownMessagesReceived() {
public void addGrpcQueueTime(int queueTime) {
grpcQueueTime.add(queueTime);
if (grpcQuantileEnable) {
for (MutableQuantiles q : grpcQueueTimeMillisQuantiles) {
for (OzoneMutableQuantiles q : grpcQueueTimeMillisQuantiles) {
if (q != null) {
q.add(queueTime);
}
Expand All @@ -171,7 +171,7 @@ public void addGrpcQueueTime(int queueTime) {
public void addGrpcProcessingTime(int processingTime) {
grpcProcessingTime.add(processingTime);
if (grpcQuantileEnable) {
for (MutableQuantiles q : grpcProcessingTimeMillisQuantiles) {
for (OzoneMutableQuantiles q : grpcProcessingTimeMillisQuantiles) {
if (q != null) {
q.add(processingTime);
}
Expand Down
8 changes: 4 additions & 4 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/metrics/OzoneAdderSampleStat.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

/**
* Helper to compute running sample stats.
* Thread safe realization for {@link org.apache.hadoop.metrics2.util.SampleStat}.
*/
public class OzoneAdderSampleStat extends SampleStat {
private final MinMax minmax = new MinMax();
Expand All @@ -45,6 +46,9 @@ public OzoneAdderSampleStat() {
total = new DoubleAdder();
}

/**
* Reset state.
*/
public void reset() {
numSamples.reset();
a0.reset();
Expand Down Expand Up @@ -239,9 +243,5 @@ public void reset(MinMax other) {
min.set(other.min());
max.set(other.max());
}
public void copyTo(MinMax other) {
other.min.set(other.min());
other.max.set(other.max());
}
}
}
Loading

0 comments on commit 0ac69a3

Please sign in to comment.