Added some more Membuster recordings to MallocBench
authorggaren@apple.com <ggaren@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 18 Apr 2014 22:31:34 +0000 (22:31 +0000)
committerggaren@apple.com <ggaren@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 18 Apr 2014 22:31:34 +0000 (22:31 +0000)
https://bugs.webkit.org/show_bug.cgi?id=131862

Reviewed by Sam Weinig.

* MallocBench/MallocBench/Benchmark.cpp:
(Benchmark::Benchmark):
(Benchmark::run):
* MallocBench/MallocBench/Benchmark.h:
* MallocBench/MallocBench/CommandLine.cpp:
* MallocBench/MallocBench/CommandLine.h:
(CommandLine::runs): Added a --runs option, so we can specify zero runs
for memory warning benchmarks. Those benchmarks want zero runs so that
they can perform a single warmup, which does not free all allocated
objects, and then see how far back to 0MB they can get. Running multiple
times would accumulate leaks, which is not representative of the
simulated scenario.

* MallocBench/MallocBench/Interpreter.cpp:
(Interpreter::Interpreter):
(Interpreter::run):
* MallocBench/MallocBench/Interpreter.h: Support not deallocating all
objects allocated during the recording, so we can do low memory warning
memory use measurements, as above.

* MallocBench/MallocBench/flickr.cpp:
(benchmark_flickr_memory_warning):
* MallocBench/MallocBench/main.cpp:
(main):
* MallocBench/MallocBench/reddit.cpp:
(benchmark_reddit_memory_warning):
* MallocBench/MallocBench/theverge.cpp:
(benchmark_theverge_memory_warning): Adopt the API above.

* MallocBench/run-malloc-benchmarks: I took a first pass at listing all
available benchmarks here. Then I commented out the benchmarks that
probably aren't reasonable to run by default.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@167511 268f45cc-cd09-0410-ab3c-d52691b4dbfc

12 files changed:
PerformanceTests/ChangeLog
PerformanceTests/MallocBench/MallocBench/Benchmark.cpp
PerformanceTests/MallocBench/MallocBench/Benchmark.h
PerformanceTests/MallocBench/MallocBench/CommandLine.cpp
PerformanceTests/MallocBench/MallocBench/CommandLine.h
PerformanceTests/MallocBench/MallocBench/Interpreter.cpp
PerformanceTests/MallocBench/MallocBench/Interpreter.h
PerformanceTests/MallocBench/MallocBench/flickr.cpp
PerformanceTests/MallocBench/MallocBench/main.cpp
PerformanceTests/MallocBench/MallocBench/reddit.cpp
PerformanceTests/MallocBench/MallocBench/theverge.cpp
PerformanceTests/MallocBench/run-malloc-benchmarks

index 9c4021e..bbde113 100644 (file)
@@ -1,5 +1,45 @@
 2014-04-18  Geoffrey Garen  <ggaren@apple.com>
 
+        Added some more Membuster recordings to MallocBench
+        https://bugs.webkit.org/show_bug.cgi?id=131862
+
+        Reviewed by Sam Weinig.
+
+        * MallocBench/MallocBench/Benchmark.cpp:
+        (Benchmark::Benchmark):
+        (Benchmark::run):
+        * MallocBench/MallocBench/Benchmark.h:
+        * MallocBench/MallocBench/CommandLine.cpp:
+        * MallocBench/MallocBench/CommandLine.h:
+        (CommandLine::runs): Added a --runs option, so we can specify zero runs
+        for memory warning benchmarks. Those benchmarks want zero runs so that
+        they can perform a single warmup, which does not free all allocated
+        objects, and then see how far back to 0MB they can get. Running multiple
+        times would accumulate leaks, which is not representative of the
+        simulated scenario.
+
+        * MallocBench/MallocBench/Interpreter.cpp:
+        (Interpreter::Interpreter):
+        (Interpreter::run):
+        * MallocBench/MallocBench/Interpreter.h: Support not deallocating all
+        objects allocated during the recording, so we can do low memory warning
+        memory use measurements, as above.
+
+        * MallocBench/MallocBench/flickr.cpp:
+        (benchmark_flickr_memory_warning):
+        * MallocBench/MallocBench/main.cpp:
+        (main):
+        * MallocBench/MallocBench/reddit.cpp:
+        (benchmark_reddit_memory_warning):
+        * MallocBench/MallocBench/theverge.cpp:
+        (benchmark_theverge_memory_warning): Adopt the API above.
+
+        * MallocBench/run-malloc-benchmarks: I took a first pass at listing all
+        available benchmarks here. Then I commented out the benchmarks that
+        probably aren't reasonable to run by default.
+
+2014-04-18  Geoffrey Garen  <ggaren@apple.com>
+
         MallocBench: removed the --measure-heap option
         https://bugs.webkit.org/show_bug.cgi?id=131854
 
index 55bef83..684fa0b 100644 (file)
@@ -119,11 +119,12 @@ static void deallocateHeap(void*** chunks, size_t heapSize, size_t chunkSize, si
     mbfree(chunks, chunkCount * sizeof(void**));
 }
 
-Benchmark::Benchmark(const string& benchmarkName, bool isParallel, size_t heapSize)
+Benchmark::Benchmark(const string& benchmarkName, bool isParallel, size_t runs, size_t heapSize)
     : m_benchmarkPair()
     , m_elapsedTime()
     , m_isParallel(isParallel)
     , m_heapSize(heapSize)
+    , m_runs(runs)
 {
     const BenchmarkPair* benchmarkPair = std::find(
         benchmarkPairs, benchmarkPairs + benchmarksPairsCount, benchmarkName);
@@ -162,7 +163,6 @@ void Benchmark::runOnce()
 
 void Benchmark::run()
 {
-    static const size_t count = 4;
     static const size_t objectSize = 32;
     static const size_t chunkSize = 1024 * 1024;
     
@@ -170,14 +170,14 @@ void Benchmark::run()
 
     runOnce(); // Warmup run.
 
-    for (size_t i = 0; i < count; ++i) {
+    for (size_t i = 0; i < m_runs; ++i) {
         double start = currentTimeMS();
         runOnce();
         double end = currentTimeMS();
         double elapsed = end - start;
         m_elapsedTime += elapsed;
     }
-    m_elapsedTime /= count;
+    m_elapsedTime /= m_runs;
 
     deallocateHeap(heap, m_heapSize, chunkSize, objectSize);
     
index b93a88b..1aee7a9 100644 (file)
@@ -59,7 +59,7 @@ public:
     static double currentTimeMS();
     static Memory currentMemoryBytes();
 
-    Benchmark(const std::string&, bool isParallel, size_t heapSize);
+    Benchmark(const std::string&, bool isParallel, size_t runs, size_t heapSize);
     
     bool isValid() { return m_benchmarkPair; }
     
@@ -76,6 +76,7 @@ private:
 
     const BenchmarkPair* m_benchmarkPair;
     bool m_isParallel;
+    size_t m_runs;
     size_t m_heapSize;
 
     Memory m_memory;
index f832c2f..b2ad0a3 100644 (file)
@@ -32,18 +32,20 @@ struct option CommandLine::longOptions[] =
     {"benchmark", required_argument, 0, 'b'},
     {"parallel", no_argument, 0, 'p'},
     {"heap", required_argument, 0, 'h'},
+    {"runs", required_argument, 0, 'r'},
     {0, 0, 0, 0}
 };
 
 CommandLine::CommandLine(int argc, char** argv)
     : m_argc(argc)
     , m_argv(argv)
-    , m_isParallel()
-    , m_heapSize()
+    , m_isParallel(false)
+    , m_heapSize(0)
+    , m_runs(4)
 {
     int optionIndex = 0;
     int ch;
-    while ((ch = getopt_long(argc, argv, "b:p:h:m", longOptions, &optionIndex)) != -1) {
+    while ((ch = getopt_long(argc, argv, "b:p:h:r", longOptions, &optionIndex)) != -1) {
         switch (ch)
         {
             case 'b':
@@ -58,6 +60,10 @@ CommandLine::CommandLine(int argc, char** argv)
                 m_heapSize = atoi(optarg) * 1024 * 1024;
                 break;
 
+            case 'r':
+                m_runs = atoi(optarg);
+                break;
+
             default:
                 break;
         }
index 14d7329..192c93e 100644 (file)
@@ -33,6 +33,7 @@ public:
     const std::string& benchmarkName() { return m_benchmarkName; }
     bool isParallel() { return m_isParallel; }
     size_t heapSize() { return m_heapSize; }
+    size_t runs() { return m_runs; }
 
     void printUsage();
 
@@ -44,4 +45,5 @@ private:
     std::string m_benchmarkName;
     bool m_isParallel;
     size_t m_heapSize;
+    size_t m_runs;
 };
index 92636ca..81d6427 100644 (file)
@@ -40,7 +40,8 @@
 
 #include "mbmalloc.h"
 
-Interpreter::Interpreter(const char* fileName)
+Interpreter::Interpreter(const char* fileName, bool shouldFreeAllObjects)
+    : m_shouldFreeAllObjects(shouldFreeAllObjects)
 {
     m_fd = open(fileName, O_RDWR, S_IRUSR | S_IWUSR);
     if (m_fd == -1)
@@ -120,6 +121,9 @@ void Interpreter::run()
     }
 
     // A recording might not free all of its allocations.
+    if (!m_shouldFreeAllObjects)
+        return;
+
     for (size_t i = 0; i < m_objects.size(); ++i) {
         if (!m_objects[i].object)
             continue;
index b812baf..f406e91 100644 (file)
@@ -30,7 +30,7 @@
 
 class Interpreter {
 public:
-    Interpreter(const char* fileName);
+    Interpreter(const char* fileName, bool shouldFreeAllObjects = true);
     ~Interpreter();
 
     void run();
@@ -40,6 +40,7 @@ private:
     struct Op { Opcode opcode; size_t slot; size_t size; };
     struct Record { void* object; size_t size; };
 
+    bool m_shouldFreeAllObjects;
     int m_fd;
     size_t m_opCount;
     std::vector<Record> m_objects;
index e337fdb..79d86f6 100644 (file)
@@ -55,7 +55,8 @@ void benchmark_flickr_memory_warning(bool isParallel)
 {
     size_t times = 1;
 
-    Interpreter interpreter("flickr_memory_warning.ops");
+    bool shouldFreeAllObjects = false;
+    Interpreter interpreter("flickr_memory_warning.ops", shouldFreeAllObjects);
     for (size_t i = 0; i < times; ++i)
         interpreter.run();
 }
index db80d44..e0b2120 100644 (file)
@@ -40,7 +40,7 @@ int main(int argc, char** argv)
         exit(1);
     }
 
-    Benchmark benchmark(commandLine.benchmarkName(), commandLine.isParallel(), commandLine.heapSize());
+    Benchmark benchmark(commandLine.benchmarkName(), commandLine.isParallel(), commandLine.runs(), commandLine.heapSize());
     if (!benchmark.isValid()) {
         cout << "Invalid benchmark: " << commandLine.benchmarkName() << endl << endl;
         benchmark.printBenchmarks();
@@ -48,12 +48,17 @@ int main(int argc, char** argv)
     }
 
     string parallel = commandLine.isParallel() ? string(" [ parallel ]") : string(" [ not parallel ]");
+
+    stringstream runs;
+    runs << " [ runs: " << commandLine.runs() << " ]";
+
     stringstream heapSize;
     if (commandLine.heapSize())
         heapSize << " [ heap: " << commandLine.heapSize() / 1024 / 1024 << "MB ]";
     else
         heapSize << " [ heap: 0MB ]";
-    cout << "Running " << commandLine.benchmarkName() << parallel << heapSize.str() << "..." << endl;
+
+    cout << "Running " << commandLine.benchmarkName() << parallel << heapSize.str() << runs.str() << "..." << endl;
     benchmark.run();
     benchmark.printReport();
         
index df993b5..8f3cc8f 100644 (file)
@@ -55,7 +55,8 @@ void benchmark_reddit_memory_warning(bool isParallel)
 {
     size_t times = 1;
 
-    Interpreter interpreter("reddit_memory_warning.ops");
+    bool shouldFreeAllObjects = false;
+    Interpreter interpreter("reddit_memory_warning.ops", shouldFreeAllObjects);
     for (size_t i = 0; i < times; ++i)
         interpreter.run();
 }
index 6ba2ece..c24d298 100644 (file)
@@ -55,7 +55,8 @@ void benchmark_theverge_memory_warning(bool isParallel)
 {
     size_t times = 1;
 
-    Interpreter interpreter("theverge_memory_warning.ops");
+    bool shouldFreeAllObjects = false;
+    Interpreter interpreter("theverge_memory_warning.ops", shouldFreeAllObjects);
     for (size_t i = 0; i < times; ++i)
         interpreter.run();
 }
index 240e13b..ff2b4ed 100755 (executable)
@@ -7,30 +7,52 @@ $binDir = "#{File.expand_path(File.dirname(__FILE__))}"
 $productDir = `perl -e 'use lib \"#{$binDir}/../../Tools/Scripts\"; use webkitdirs; print productDir()'`
 
 $benchmarks = [
+    # Single-threaded benchmarks.
     "churn",
-    "churn --parallel",
     "list_allocate",
-    "list_allocate --parallel",
     "tree_allocate",
-    "tree_allocate --parallel",
     "tree_churn",
-    "tree_churn --parallel",
-    "facebook",
-    "facebook --parallel",
     "fragment",
-    "fragment --parallel",
     "fragment_iterate",
-    "fragment_iterate --parallel",
+    "medium",
+    "big",
+
+    # Benchmarks based on browser recordings.
+    "facebook",
+    "reddit",
+    "flickr",
+    "theverge",
+
+    # Multi-threaded benchmark variants.
     "message_one",
     "message_many",
-    "medium",
+    "churn --parallel",
+    "list_allocate --parallel",
+    "tree_allocate --parallel",
+    "tree_churn --parallel",
+    # "facebook --parallel",
+    # "reddit --parallel",
+    # "flickr --parallel",
+    # "theverge --parallel",
+    "fragment --parallel",
+    "fragment_iterate --parallel",
+
+    # These tests often crash TCMalloc: <rdar://problem/13657137>.
     "medium --parallel",
-    "big",
     "big --parallel",
+
+    # Enable these tests to test memory footprint. The way they run is not
+    # really compatible with throughput testing.
+    # "reddit_memory_warning --runs 0",
+    # "flickr_memory_warning --runs 0",
+    # "theverge_memory_warning --runs 0",
+
+    # Enable this test to test shrinking back down from a large heap while a process remains active.
+    # The way it runs is not really compatible with throughput testing.
+    # "balloon"
 ]
 
 $heap = 0
-$measureHeap = false
 
 def usage
        puts "run-malloc-benchmarks [options] <Name:/path/to/dylib> [<Name:/path/to/dylib>]"
@@ -175,7 +197,6 @@ def parseOptions
     GetoptLong.new(
         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
-        ['--measure-heap', GetoptLong::NO_ARGUMENT],
         ['--help', GetoptLong::NO_ARGUMENT],
     ).each {
         | opt, arg |
@@ -184,8 +205,6 @@ def parseOptions
             $benchmarks = [ arg ]
         when '--heap'
             $heap = arg
-        when '--measure-heap'
-            $measureHeap = true
         when '--help'
             usage
             exit 1
@@ -243,12 +262,12 @@ def runBenchmarks(dylibs)
         dylibs.each {
             | dylib |
 
-            $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                "
+            $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                                "
             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
             if dylib.name == "NanoMalloc"
                 env += "MallocNanoZone=1 "
             end
-            input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap} #{$measureHeap ? '--measure-heap' : ''}"
+            input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap}}"
             output =`#{input}`
             splitOutput = output.split("\n")
 
@@ -267,9 +286,9 @@ def printResults(dylibs, results)
         print
         print lpad("", fieldSize)
         print lpad(dylibs[0].name, fieldSize)
-        if dylibs[1]
+        if dylibs.length > 1
             print lpad(dylibs[1].name, fieldSize)
-            print lpad("Δ", fieldSize * 1.25)
+            print lpad("Δ", fieldSize)
         end
         print "\n"
     end
@@ -285,7 +304,7 @@ def printResults(dylibs, results)
             if results[0][1]
                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
-                print lpad(compareFunction.call(means[0], means[1]), fieldSize * 1.25)
+                print lpad(compareFunction.call(means[0], means[1]), fieldSize)
             end
 
             print "\n"
@@ -304,7 +323,7 @@ def printResults(dylibs, results)
 
             if stats[1]
                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
-                print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize * 1.25)
+                print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
             end
 
             print "\n"