1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.perftests.utils;
18 
19 import android.app.Activity;
20 import android.app.Instrumentation;
21 import android.os.Bundle;
22 import android.os.Debug;
23 import android.util.Log;
24 
25 import androidx.test.InstrumentationRegistry;
26 
27 import java.io.File;
28 import java.util.ArrayList;
29 import java.util.concurrent.TimeUnit;
30 
31 /**
32  * Provides a benchmark framework.
33  *
34  * Example usage:
35  * // Executes the code while keepRunning returning true.
36  *
37  * public void sampleMethod() {
38  *     BenchmarkState state = new BenchmarkState();
39  *
40  *     int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
41  *     while (state.keepRunning()) {
42  *         int[] dest = new int[src.length];
43  *         System.arraycopy(src, 0, dest, 0, src.length);
44  *     }
45  *     System.out.println(state.summaryLine());
46  * }
47  */
48 public final class BenchmarkState {
49 
50     private static final String TAG = "BenchmarkState";
51     private static final boolean ENABLE_PROFILING = false;
52 
53     private static final int NOT_STARTED = 0;  // The benchmark has not started yet.
54     private static final int WARMUP = 1; // The benchmark is warming up.
55     private static final int RUNNING = 2;  // The benchmark is running.
56     private static final int RUNNING_CUSTOMIZED = 3;  // Running for customized measurement.
57     private static final int FINISHED = 4;  // The benchmark has stopped.
58 
59     private int mState = NOT_STARTED;  // Current benchmark state.
60 
61     private static final long WARMUP_DURATION_NS = ms2ns(250); // warm-up for at least 250ms
62     private static final int WARMUP_MIN_ITERATIONS = 16; // minimum iterations to warm-up for
63 
64     // TODO: Tune these values.
65     private static final long TARGET_TEST_DURATION_NS = ms2ns(500); // target testing for 500 ms
66     private static final int MAX_TEST_ITERATIONS = 1000000;
67     private static final int MIN_TEST_ITERATIONS = 10;
68     private static final int REPEAT_COUNT = 5;
69 
70     private long mStartTimeNs = 0;  // Previously captured System.nanoTime().
71     private boolean mPaused;
72     private long mPausedTimeNs = 0; // The System.nanoTime() when the pauseTiming() is called.
73     private long mPausedDurationNs = 0;  // The duration of paused state in nano sec.
74 
75     private int mIteration = 0;
76     private int mMaxIterations = 0;
77 
78     private int mRepeatCount = 0;
79 
80     /**
81      * Additional iteration that used to apply customized measurement. The result during these
82      * iterations won't be counted into {@link #mStats}.
83      */
84     private int mMaxCustomizedIterations;
85     private int mCustomizedIterations;
86     private CustomizedIterationListener mCustomizedIterationListener;
87 
88     // Statistics. These values will be filled when the benchmark has finished.
89     // The computation needs double precision, but long int is fine for final reporting.
90     private Stats mStats;
91 
92     // Individual duration in nano seconds.
93     private ArrayList<Long> mResults = new ArrayList<>();
94 
ms2ns(long ms)95     private static final long ms2ns(long ms) {
96         return TimeUnit.MILLISECONDS.toNanos(ms);
97     }
98 
99     // Stops the benchmark timer.
100     // This method can be called only when the timer is running.
pauseTiming()101     public void pauseTiming() {
102         if (mPaused) {
103             throw new IllegalStateException(
104                     "Unable to pause the benchmark. The benchmark has already paused.");
105         }
106         mPausedTimeNs = System.nanoTime();
107         mPaused = true;
108     }
109 
110     // Starts the benchmark timer.
111     // This method can be called only when the timer is stopped.
resumeTiming()112     public void resumeTiming() {
113         if (!mPaused) {
114             throw new IllegalStateException(
115                     "Unable to resume the benchmark. The benchmark is already running.");
116         }
117         mPausedDurationNs += System.nanoTime() - mPausedTimeNs;
118         mPausedTimeNs = 0;
119         mPaused = false;
120     }
121 
122     /**
123      * This is used to run the benchmark with more information by enabling some debug mechanism but
124      * we don't want to account the special runs (slower) in the stats report.
125      */
setCustomizedIterations(int iterations, CustomizedIterationListener listener)126     public void setCustomizedIterations(int iterations, CustomizedIterationListener listener) {
127         mMaxCustomizedIterations = iterations;
128         mCustomizedIterationListener = listener;
129     }
130 
beginWarmup()131     private void beginWarmup() {
132         mStartTimeNs = System.nanoTime();
133         mIteration = 0;
134         mState = WARMUP;
135     }
136 
beginBenchmark(long warmupDuration, int iterations)137     private void beginBenchmark(long warmupDuration, int iterations) {
138         if (ENABLE_PROFILING) {
139             File f = new File(InstrumentationRegistry.getContext().getDataDir(), "benchprof");
140             Log.d(TAG, "Tracing to: " + f.getAbsolutePath());
141             Debug.startMethodTracingSampling(f.getAbsolutePath(), 16 * 1024 * 1024, 100);
142         }
143         mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations));
144         mMaxIterations = Math.min(MAX_TEST_ITERATIONS,
145                 Math.max(mMaxIterations, MIN_TEST_ITERATIONS));
146         mPausedDurationNs = 0;
147         mIteration = 0;
148         mRepeatCount = 0;
149         mState = RUNNING;
150         mStartTimeNs = System.nanoTime();
151     }
152 
startNextTestRun()153     private boolean startNextTestRun() {
154         final long currentTime = System.nanoTime();
155         mResults.add((currentTime - mStartTimeNs - mPausedDurationNs) / mMaxIterations);
156         mRepeatCount++;
157         if (mRepeatCount >= REPEAT_COUNT) {
158             if (ENABLE_PROFILING) {
159                 Debug.stopMethodTracing();
160             }
161             mStats = new Stats(mResults);
162             if (mMaxCustomizedIterations > 0 && mCustomizedIterationListener != null) {
163                 mState = RUNNING_CUSTOMIZED;
164                 mCustomizedIterationListener.onStart(mCustomizedIterations);
165                 return true;
166             }
167             mState = FINISHED;
168             return false;
169         }
170         mPausedDurationNs = 0;
171         mIteration = 0;
172         mStartTimeNs = System.nanoTime();
173         return true;
174     }
175 
176     /**
177      * Judges whether the benchmark needs more samples.
178      *
179      * For the usage, see class comment.
180      */
keepRunning()181     public boolean keepRunning() {
182         switch (mState) {
183             case NOT_STARTED:
184                 beginWarmup();
185                 return true;
186             case WARMUP:
187                 mIteration++;
188                 // Only check nanoTime on every iteration in WARMUP since we
189                 // don't yet have a target iteration count.
190                 final long duration = System.nanoTime() - mStartTimeNs;
191                 if (mIteration >= WARMUP_MIN_ITERATIONS && duration >= WARMUP_DURATION_NS) {
192                     beginBenchmark(duration, mIteration);
193                 }
194                 return true;
195             case RUNNING:
196                 mIteration++;
197                 if (mIteration >= mMaxIterations) {
198                     return startNextTestRun();
199                 }
200                 if (mPaused) {
201                     throw new IllegalStateException(
202                             "Benchmark step finished with paused state. " +
203                             "Resume the benchmark before finishing each step.");
204                 }
205                 return true;
206             case RUNNING_CUSTOMIZED:
207                 mCustomizedIterationListener.onFinished(mCustomizedIterations);
208                 mCustomizedIterations++;
209                 if (mCustomizedIterations >= mMaxCustomizedIterations) {
210                     mState = FINISHED;
211                     return false;
212                 }
213                 mCustomizedIterationListener.onStart(mCustomizedIterations);
214                 return true;
215             case FINISHED:
216                 throw new IllegalStateException("The benchmark has finished.");
217             default:
218                 throw new IllegalStateException("The benchmark is in unknown state.");
219         }
220     }
221 
mean()222     private long mean() {
223         if (mState != FINISHED) {
224             throw new IllegalStateException("The benchmark hasn't finished");
225         }
226         return (long) mStats.getMean();
227     }
228 
median()229     private long median() {
230         if (mState != FINISHED) {
231             throw new IllegalStateException("The benchmark hasn't finished");
232         }
233         return mStats.getMedian();
234     }
235 
min()236     private long min() {
237         if (mState != FINISHED) {
238             throw new IllegalStateException("The benchmark hasn't finished");
239         }
240         return mStats.getMin();
241     }
242 
standardDeviation()243     private long standardDeviation() {
244         if (mState != FINISHED) {
245             throw new IllegalStateException("The benchmark hasn't finished");
246         }
247         return (long) mStats.getStandardDeviation();
248     }
249 
summaryLine()250     private String summaryLine() {
251         StringBuilder sb = new StringBuilder();
252         sb.append("Summary: ");
253         sb.append("median=").append(median()).append("ns, ");
254         sb.append("mean=").append(mean()).append("ns, ");
255         sb.append("min=").append(min()).append("ns, ");
256         sb.append("sigma=").append(standardDeviation()).append(", ");
257         sb.append("iteration=").append(mResults.size()).append(", ");
258         // print out the first few iterations' number for double checking.
259         int sampleNumber = Math.min(mResults.size(), 16);
260         for (int i = 0; i < sampleNumber; i++) {
261             sb.append("No ").append(i).append(" result is ").append(mResults.get(i)).append(", ");
262         }
263         return sb.toString();
264     }
265 
sendFullStatusReport(Instrumentation instrumentation, String key)266     public void sendFullStatusReport(Instrumentation instrumentation, String key) {
267         Log.i(TAG, key + summaryLine());
268         Bundle status = new Bundle();
269         status.putLong(key + "_median (ns)", median());
270         status.putLong(key + "_mean (ns)", mean());
271         status.putLong(key + "_min (ns)", min());
272         status.putLong(key + "_standardDeviation", standardDeviation());
273         instrumentation.sendStatus(Activity.RESULT_OK, status);
274     }
275 
276     /** The interface to receive the events of customized iteration. */
277     public interface CustomizedIterationListener {
278         /** The customized iteration starts. */
onStart(int iteration)279         void onStart(int iteration);
280 
281         /** The customized iteration finished. */
onFinished(int iteration)282         void onFinished(int iteration);
283     }
284 }
285