Added some more Membuster recordings to MallocBench
authorggaren@apple.com <ggaren@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 18 Apr 2014 22:31:34 +0000 (22:31 +0000)
committerggaren@apple.com <ggaren@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 18 Apr 2014 22:31:34 +0000 (22:31 +0000)
https://bugs.webkit.org/show_bug.cgi?id=131862

Reviewed by Sam Weinig.

* MallocBench/MallocBench/Benchmark.cpp:
(Benchmark::Benchmark):
(Benchmark::run):
* MallocBench/MallocBench/Benchmark.h:
* MallocBench/MallocBench/CommandLine.cpp:
* MallocBench/MallocBench/CommandLine.h:
(CommandLine::runs): Added a --runs option, so we can specify zero runs
for memory warning benchmarks. Those benchmarks want zero runs so that
they can perform a single warmup, which does not free all allocated
objects, and then see how far back to 0MB they can get. Running multiple
times would accumulate leaks, which is not representative of the
simulated scenario.

* MallocBench/MallocBench/Interpreter.cpp:
(Interpreter::Interpreter):
(Interpreter::run):
* MallocBench/MallocBench/Interpreter.h: Support not deallocating all
objects allocated during the recording, so we can do low memory warning
memory use measurements, as above.

* MallocBench/MallocBench/flickr.cpp:
(benchmark_flickr_memory_warning):
* MallocBench/MallocBench/main.cpp:
(main):
* MallocBench/MallocBench/reddit.cpp:
(benchmark_reddit_memory_warning):
* MallocBench/MallocBench/theverge.cpp:
(benchmark_theverge_memory_warning): Adopt the API above.

* MallocBench/run-malloc-benchmarks: I took a first pass at listing all
available benchmarks here. Then I commented out the benchmarks that
probably aren't reasonable to run by default.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@167511 268f45cc-cd09-0410-ab3c-d52691b4dbfc

12 files changed:
PerformanceTests/ChangeLog
PerformanceTests/MallocBench/MallocBench/Benchmark.cpp
PerformanceTests/MallocBench/MallocBench/Benchmark.h
PerformanceTests/MallocBench/MallocBench/CommandLine.cpp
PerformanceTests/MallocBench/MallocBench/CommandLine.h
PerformanceTests/MallocBench/MallocBench/Interpreter.cpp
PerformanceTests/MallocBench/MallocBench/Interpreter.h
PerformanceTests/MallocBench/MallocBench/flickr.cpp
PerformanceTests/MallocBench/MallocBench/main.cpp
PerformanceTests/MallocBench/MallocBench/reddit.cpp
PerformanceTests/MallocBench/MallocBench/theverge.cpp
PerformanceTests/MallocBench/run-malloc-benchmarks

index 9c4021eb677e474d88b3e15e2204e756cdc402b8..bbde113d6e2ac338c0a59f6851810abe08faaafc 100644 (file)
@@ -1,3 +1,43 @@
+2014-04-18  Geoffrey Garen  <ggaren@apple.com>
+
+        Added some more Membuster recordings to MallocBench
+        https://bugs.webkit.org/show_bug.cgi?id=131862
+
+        Reviewed by Sam Weinig.
+
+        * MallocBench/MallocBench/Benchmark.cpp:
+        (Benchmark::Benchmark):
+        (Benchmark::run):
+        * MallocBench/MallocBench/Benchmark.h:
+        * MallocBench/MallocBench/CommandLine.cpp:
+        * MallocBench/MallocBench/CommandLine.h:
+        (CommandLine::runs): Added a --runs option, so we can specify zero runs
+        for memory warning benchmarks. Those benchmarks want zero runs so that
+        they can perform a single warmup, which does not free all allocated
+        objects, and then see how far back to 0MB they can get. Running multiple
+        times would accumulate leaks, which is not representative of the
+        simulated scenario.
+
+        * MallocBench/MallocBench/Interpreter.cpp:
+        (Interpreter::Interpreter):
+        (Interpreter::run):
+        * MallocBench/MallocBench/Interpreter.h: Support not deallocating all
+        objects allocated during the recording, so we can do low memory warning
+        memory use measurements, as above.
+
+        * MallocBench/MallocBench/flickr.cpp:
+        (benchmark_flickr_memory_warning):
+        * MallocBench/MallocBench/main.cpp:
+        (main):
+        * MallocBench/MallocBench/reddit.cpp:
+        (benchmark_reddit_memory_warning):
+        * MallocBench/MallocBench/theverge.cpp:
+        (benchmark_theverge_memory_warning): Adopt the API above.
+
+        * MallocBench/run-malloc-benchmarks: I took a first pass at listing all
+        available benchmarks here. Then I commented out the benchmarks that
+        probably aren't reasonable to run by default.
+
 2014-04-18  Geoffrey Garen  <ggaren@apple.com>
 
         MallocBench: removed the --measure-heap option
index 55bef83301ae4e51821d007a9e16ee291dfa620c..684fa0ba3e66cc94bc54041b670a768e7988b7b5 100644 (file)
@@ -119,11 +119,12 @@ static void deallocateHeap(void*** chunks, size_t heapSize, size_t chunkSize, si
     mbfree(chunks, chunkCount * sizeof(void**));
 }
 
-Benchmark::Benchmark(const string& benchmarkName, bool isParallel, size_t heapSize)
+Benchmark::Benchmark(const string& benchmarkName, bool isParallel, size_t runs, size_t heapSize)
     : m_benchmarkPair()
     , m_elapsedTime()
     , m_isParallel(isParallel)
     , m_heapSize(heapSize)
+    , m_runs(runs)
 {
     const BenchmarkPair* benchmarkPair = std::find(
         benchmarkPairs, benchmarkPairs + benchmarksPairsCount, benchmarkName);
@@ -162,7 +163,6 @@ void Benchmark::runOnce()
 
 void Benchmark::run()
 {
-    static const size_t count = 4;
     static const size_t objectSize = 32;
     static const size_t chunkSize = 1024 * 1024;
     
@@ -170,14 +170,14 @@ void Benchmark::run()
 
     runOnce(); // Warmup run.
 
-    for (size_t i = 0; i < count; ++i) {
+    for (size_t i = 0; i < m_runs; ++i) {
         double start = currentTimeMS();
         runOnce();
         double end = currentTimeMS();
         double elapsed = end - start;
         m_elapsedTime += elapsed;
     }
-    m_elapsedTime /= count;
+    m_elapsedTime /= m_runs;
 
     deallocateHeap(heap, m_heapSize, chunkSize, objectSize);
     
index b93a88b77bb4ae0cb8253f1988f3415db267fd65..1aee7a9408047793426003fd7854e0c3d759b5da 100644 (file)
@@ -59,7 +59,7 @@ public:
     static double currentTimeMS();
     static Memory currentMemoryBytes();
 
-    Benchmark(const std::string&, bool isParallel, size_t heapSize);
+    Benchmark(const std::string&, bool isParallel, size_t runs, size_t heapSize);
     
     bool isValid() { return m_benchmarkPair; }
     
@@ -76,6 +76,7 @@ private:
 
     const BenchmarkPair* m_benchmarkPair;
     bool m_isParallel;
+    size_t m_runs;
     size_t m_heapSize;
 
     Memory m_memory;
index f832c2f0fc7400021a5f37cc474ddf044e5a931d..b2ad0a37fa5362f7df42023dfea525f4d12c5075 100644 (file)
@@ -32,18 +32,20 @@ struct option CommandLine::longOptions[] =
     {"benchmark", required_argument, 0, 'b'},
     {"parallel", no_argument, 0, 'p'},
     {"heap", required_argument, 0, 'h'},
+    {"runs", required_argument, 0, 'r'},
     {0, 0, 0, 0}
 };
 
 CommandLine::CommandLine(int argc, char** argv)
     : m_argc(argc)
     , m_argv(argv)
-    , m_isParallel()
-    , m_heapSize()
+    , m_isParallel(false)
+    , m_heapSize(0)
+    , m_runs(4)
 {
     int optionIndex = 0;
     int ch;
-    while ((ch = getopt_long(argc, argv, "b:p:h:m", longOptions, &optionIndex)) != -1) {
+    while ((ch = getopt_long(argc, argv, "b:p:h:r", longOptions, &optionIndex)) != -1) {
         switch (ch)
         {
             case 'b':
@@ -58,6 +60,10 @@ CommandLine::CommandLine(int argc, char** argv)
                 m_heapSize = atoi(optarg) * 1024 * 1024;
                 break;
 
+            case 'r':
+                m_runs = atoi(optarg);
+                break;
+
             default:
                 break;
         }
index 14d7329f8f6c459c07cce4834c6ec3e650ea67f5..192c93e69b8f377f9bdb33fcb45a0e003fc9724d 100644 (file)
@@ -33,6 +33,7 @@ public:
     const std::string& benchmarkName() { return m_benchmarkName; }
     bool isParallel() { return m_isParallel; }
     size_t heapSize() { return m_heapSize; }
+    size_t runs() { return m_runs; }
 
     void printUsage();
 
@@ -44,4 +45,5 @@ private:
     std::string m_benchmarkName;
     bool m_isParallel;
     size_t m_heapSize;
+    size_t m_runs;
 };
index 92636caff651fec26a125fdca7fb5dcd728880a9..81d6427bfcc51325b2ed4a5f818ef87a0233cc18 100644 (file)
@@ -40,7 +40,8 @@
 
 #include "mbmalloc.h"
 
-Interpreter::Interpreter(const char* fileName)
+Interpreter::Interpreter(const char* fileName, bool shouldFreeAllObjects)
+    : m_shouldFreeAllObjects(shouldFreeAllObjects)
 {
     m_fd = open(fileName, O_RDWR, S_IRUSR | S_IWUSR);
     if (m_fd == -1)
@@ -120,6 +121,9 @@ void Interpreter::run()
     }
 
     // A recording might not free all of its allocations.
+    if (!m_shouldFreeAllObjects)
+        return;
+
     for (size_t i = 0; i < m_objects.size(); ++i) {
         if (!m_objects[i].object)
             continue;
index b812baf3508d670dc897544db7ee160cbba10e64..f406e9130bf1af5252976353647903421c0383e3 100644 (file)
@@ -30,7 +30,7 @@
 
 class Interpreter {
 public:
-    Interpreter(const char* fileName);
+    Interpreter(const char* fileName, bool shouldFreeAllObjects = true);
     ~Interpreter();
 
     void run();
@@ -40,6 +40,7 @@ private:
     struct Op { Opcode opcode; size_t slot; size_t size; };
     struct Record { void* object; size_t size; };
 
+    bool m_shouldFreeAllObjects;
     int m_fd;
     size_t m_opCount;
     std::vector<Record> m_objects;
index e337fdbf61f48d73f050ba44fb55ce2d3267e58c..79d86f6354ed8f077882e1352cae92ba87e1518a 100644 (file)
@@ -55,7 +55,8 @@ void benchmark_flickr_memory_warning(bool isParallel)
 {
     size_t times = 1;
 
-    Interpreter interpreter("flickr_memory_warning.ops");
+    bool shouldFreeAllObjects = false;
+    Interpreter interpreter("flickr_memory_warning.ops", shouldFreeAllObjects);
     for (size_t i = 0; i < times; ++i)
         interpreter.run();
 }
index db80d44e9e99995ccaf085fe7f46b76a9063aee5..e0b2120a8d6c2b189588bfad72eff8a8d38460ea 100644 (file)
@@ -40,7 +40,7 @@ int main(int argc, char** argv)
         exit(1);
     }
 
-    Benchmark benchmark(commandLine.benchmarkName(), commandLine.isParallel(), commandLine.heapSize());
+    Benchmark benchmark(commandLine.benchmarkName(), commandLine.isParallel(), commandLine.runs(), commandLine.heapSize());
     if (!benchmark.isValid()) {
         cout << "Invalid benchmark: " << commandLine.benchmarkName() << endl << endl;
         benchmark.printBenchmarks();
@@ -48,12 +48,17 @@ int main(int argc, char** argv)
     }
 
     string parallel = commandLine.isParallel() ? string(" [ parallel ]") : string(" [ not parallel ]");
+
+    stringstream runs;
+    runs << " [ runs: " << commandLine.runs() << " ]";
+
     stringstream heapSize;
     if (commandLine.heapSize())
         heapSize << " [ heap: " << commandLine.heapSize() / 1024 / 1024 << "MB ]";
     else
         heapSize << " [ heap: 0MB ]";
-    cout << "Running " << commandLine.benchmarkName() << parallel << heapSize.str() << "..." << endl;
+
+    cout << "Running " << commandLine.benchmarkName() << parallel << heapSize.str() << runs.str() << "..." << endl;
     benchmark.run();
     benchmark.printReport();
         
index df993b50aaf7156004fe0c3f33f48d7f3a2418c2..8f3cc8fb18278c6a717978305db7965d7d5124eb 100644 (file)
@@ -55,7 +55,8 @@ void benchmark_reddit_memory_warning(bool isParallel)
 {
     size_t times = 1;
 
-    Interpreter interpreter("reddit_memory_warning.ops");
+    bool shouldFreeAllObjects = false;
+    Interpreter interpreter("reddit_memory_warning.ops", shouldFreeAllObjects);
     for (size_t i = 0; i < times; ++i)
         interpreter.run();
 }
index 6ba2ece0b3b3c43f1d42dcdd77b1e930ee59b90c..c24d2982572b1ce02ceeb268a862c829584e6879 100644 (file)
@@ -55,7 +55,8 @@ void benchmark_theverge_memory_warning(bool isParallel)
 {
     size_t times = 1;
 
-    Interpreter interpreter("theverge_memory_warning.ops");
+    bool shouldFreeAllObjects = false;
+    Interpreter interpreter("theverge_memory_warning.ops", shouldFreeAllObjects);
     for (size_t i = 0; i < times; ++i)
         interpreter.run();
 }
index 240e13b90686576dc3923c48412abd861cb4bc35..ff2b4ed00b7191200e87eea0b44043f077e7e8b0 100755 (executable)
@@ -7,30 +7,52 @@ $binDir = "#{File.expand_path(File.dirname(__FILE__))}"
 $productDir = `perl -e 'use lib \"#{$binDir}/../../Tools/Scripts\"; use webkitdirs; print productDir()'`
 
 $benchmarks = [
+    # Single-threaded benchmarks.
     "churn",
-    "churn --parallel",
     "list_allocate",
-    "list_allocate --parallel",
     "tree_allocate",
-    "tree_allocate --parallel",
     "tree_churn",
-    "tree_churn --parallel",
-    "facebook",
-    "facebook --parallel",
     "fragment",
-    "fragment --parallel",
     "fragment_iterate",
-    "fragment_iterate --parallel",
+    "medium",
+    "big",
+
+    # Benchmarks based on browser recordings.
+    "facebook",
+    "reddit",
+    "flickr",
+    "theverge",
+
+    # Multi-threaded benchmark variants.
     "message_one",
     "message_many",
-    "medium",
+    "churn --parallel",
+    "list_allocate --parallel",
+    "tree_allocate --parallel",
+    "tree_churn --parallel",
+    # "facebook --parallel",
+    # "reddit --parallel",
+    # "flickr --parallel",
+    # "theverge --parallel",
+    "fragment --parallel",
+    "fragment_iterate --parallel",
+
+    # These tests often crash TCMalloc: <rdar://problem/13657137>.
     "medium --parallel",
-    "big",
     "big --parallel",
+
+    # Enable these tests to test memory footprint. The way they run is not
+    # really compatible with throughput testing.
+    # "reddit_memory_warning --runs 0",
+    # "flickr_memory_warning --runs 0",
+    # "theverge_memory_warning --runs 0",
+
+    # Enable this test to test shrinking back down from a large heap while a process remains active.
+    # The way it runs is not really compatible with throughput testing.
+    # "balloon"
 ]
 
 $heap = 0
-$measureHeap = false
 
 def usage
        puts "run-malloc-benchmarks [options] <Name:/path/to/dylib> [<Name:/path/to/dylib>]"
@@ -175,7 +197,6 @@ def parseOptions
     GetoptLong.new(
         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
-        ['--measure-heap', GetoptLong::NO_ARGUMENT],
         ['--help', GetoptLong::NO_ARGUMENT],
     ).each {
         | opt, arg |
@@ -184,8 +205,6 @@ def parseOptions
             $benchmarks = [ arg ]
         when '--heap'
             $heap = arg
-        when '--measure-heap'
-            $measureHeap = true
         when '--help'
             usage
             exit 1
@@ -243,12 +262,12 @@ def runBenchmarks(dylibs)
         dylibs.each {
             | dylib |
 
-            $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                "
+            $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                                "
             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
             if dylib.name == "NanoMalloc"
                 env += "MallocNanoZone=1 "
             end
-            input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap} #{$measureHeap ? '--measure-heap' : ''}"
+            input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap}}"
             output =`#{input}`
             splitOutput = output.split("\n")
 
@@ -267,9 +286,9 @@ def printResults(dylibs, results)
         print
         print lpad("", fieldSize)
         print lpad(dylibs[0].name, fieldSize)
-        if dylibs[1]
+        if dylibs.length > 1
             print lpad(dylibs[1].name, fieldSize)
-            print lpad("Δ", fieldSize * 1.25)
+            print lpad("Δ", fieldSize)
         end
         print "\n"
     end
@@ -285,7 +304,7 @@ def printResults(dylibs, results)
             if results[0][1]
                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
-                print lpad(compareFunction.call(means[0], means[1]), fieldSize * 1.25)
+                print lpad(compareFunction.call(means[0], means[1]), fieldSize)
             end
 
             print "\n"
@@ -304,7 +323,7 @@ def printResults(dylibs, results)
 
             if stats[1]
                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
-                print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize * 1.25)
+                print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
             end
 
             print "\n"