From: aroben Date: Tue, 26 Jun 2007 00:43:56 +0000 (+0000) Subject: Land pdevenv and supporting scripts/programs X-Git-Url: http://git.webkit.org/?p=WebKit-https.git;a=commitdiff_plain;h=96d60656162f1233624abc95cf627c26b89666ca;ds=sidebyside Land pdevenv and supporting scripts/programs pdevenv is a script that will open an instance of Visual Studio that can compile multiple files in parallel, similar to make -jN. It uses the following scripts/programs to accomplish this: CLWrapper: Compiles to vcbin/cl.exe. Calls Scripts/parallelcl. parallelcl: Actually performs the parallel compilation by forking multiple instances of the Microsoft-supplied cl.exe. MIDLWrapper: Compiles to vcbin/midl.exe. Calls through to the Microsoft-supplied midl.exe. This avoids having to invoke perl for every invocation of midl.exe, which would be quite slow. Rubberstamped by Sam. * CLWrapper/CLWrapper.cpp: Added. (wmain): * CLWrapper/CLWrapper.sln: Added. * CLWrapper/CLWrapper.vcproj: Added. * MIDLWrapper/MIDLWrapper.cpp: Added. (wmain): * MIDLWrapper/MIDLWrapper.sln: Added. * MIDLWrapper/MIDLWrapper.vcproj: Added. * Scripts/parallelcl: Added. * Scripts/pdevenv: Added. * vcbin/cl.exe: Added. * vcbin/midl.exe: Added. git-svn-id: https://svn.webkit.org/repository/webkit/trunk@23775 268f45cc-cd09-0410-ab3c-d52691b4dbfc --- diff --git a/WebKitTools/CLWrapper/CLWrapper.cpp b/WebKitTools/CLWrapper/CLWrapper.cpp new file mode 100644 index 0000000..7d41f2b --- /dev/null +++ b/WebKitTools/CLWrapper/CLWrapper.cpp @@ -0,0 +1,52 @@ +// CLWrapper.cpp : Calls the perl script parallelcl to perform parallel compilation + +#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers +#include +#include +#include +#include + +using namespace std; + +int wmain(int argc, wchar_t* argv[]) +{ + const int numArgs = 3; + +#ifndef NDEBUG + fwprintf(stderr, L"######### im in ur IDE, compiling ur c0des ########\n"); +#endif + + wstring** args = new wstring*[numArgs]; + + args[0] = new wstring(L"sh"); + args[1] = new wstring(L"-c"); + + args[2] = new wstring(L"\"parallelcl"); + for (int i = 1; i < argc; ++i) { + args[2]->append(L" '"); + args[2]->append(argv[i]); + if (i < argc - 1) + args[2]->append(L"' "); + else + args[2]->append(L"'"); + } + args[2]->append(L"\""); + + for (unsigned i = 0; i < args[2]->length(); i++) { + if (args[2]->at(i) == '\\') + args[2]->at(i) = '/'; + } + + wchar_t** newArgv = new wchar_t*[numArgs + 1]; + for (int i = 0; i < numArgs; i++) + newArgv[i] = (wchar_t*)args[i]->c_str(); + + newArgv[numArgs] = 0; + +#ifndef NDEBUG + fwprintf(stderr, L"exec(\"%s\", \"%s\", \"%s\", \"%s\")\n", L"sh", newArgv[0], newArgv[1], newArgv[2]); +#endif + + return _wspawnvp(_P_WAIT, L"sh", newArgv); +} + diff --git a/WebKitTools/CLWrapper/CLWrapper.sln b/WebKitTools/CLWrapper/CLWrapper.sln new file mode 100644 index 0000000..7501ea6 --- /dev/null +++ b/WebKitTools/CLWrapper/CLWrapper.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 9.00 +# Visual Studio 2005 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CLWrapper", "CLWrapper.vcproj", "{230BF635-9BD8-434A-8857-0B096EBC7233}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {230BF635-9BD8-434A-8857-0B096EBC7233}.Debug|Win32.ActiveCfg = Debug|Win32 + {230BF635-9BD8-434A-8857-0B096EBC7233}.Debug|Win32.Build.0 = Debug|Win32 + {230BF635-9BD8-434A-8857-0B096EBC7233}.Release|Win32.ActiveCfg = Release|Win32 + {230BF635-9BD8-434A-8857-0B096EBC7233}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/WebKitTools/CLWrapper/CLWrapper.vcproj b/WebKitTools/CLWrapper/CLWrapper.vcproj new file mode 100644 index 0000000..844d72a --- /dev/null +++ b/WebKitTools/CLWrapper/CLWrapper.vcproj @@ -0,0 +1,199 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/WebKitTools/ChangeLog b/WebKitTools/ChangeLog index a56c806..323d9b0 100644 --- a/WebKitTools/ChangeLog +++ b/WebKitTools/ChangeLog @@ -1,3 +1,35 @@ +2007-06-25 Adam Roben + + Land pdevenv and supporting scripts/programs + + pdevenv is a script that will open an instance of Visual Studio that + can compile multiple files in parallel, similar to make -jN. It uses + the following scripts/programs to accomplish this: + + CLWrapper: Compiles to vcbin/cl.exe. Calls Scripts/parallelcl. + + parallelcl: Actually performs the parallel compilation by forking + multiple instances of the Microsoft-supplied cl.exe. + + MIDLWrapper: Compiles to vcbin/midl.exe. Calls through to the + Microsoft-supplied midl.exe. This avoids having to invoke perl for + every invocation of midl.exe, which would be quite slow. + + Rubberstamped by Sam. + + * CLWrapper/CLWrapper.cpp: Added. + (wmain): + * CLWrapper/CLWrapper.sln: Added. + * CLWrapper/CLWrapper.vcproj: Added. + * MIDLWrapper/MIDLWrapper.cpp: Added. + (wmain): + * MIDLWrapper/MIDLWrapper.sln: Added. + * MIDLWrapper/MIDLWrapper.vcproj: Added. + * Scripts/parallelcl: Added. + * Scripts/pdevenv: Added. + * vcbin/cl.exe: Added. + * vcbin/midl.exe: Added. + 2007-06-23 Adam Roben Land num-cpus for the Windows build. diff --git a/WebKitTools/MIDLWrapper/MIDLWrapper.cpp b/WebKitTools/MIDLWrapper/MIDLWrapper.cpp new file mode 100644 index 0000000..2132af8 --- /dev/null +++ b/WebKitTools/MIDLWrapper/MIDLWrapper.cpp @@ -0,0 +1,86 @@ +// MIDLWrapper.cpp : Just calls the built-in midl.exe with the given arguments. + +#define WIN32_LEAN_AND_MEAN // Exclude rarely-used stuff from Windows headers +#include +#include +#include +#include + +using namespace std; + +int wmain(int argc, wchar_t* argv[], wchar_t* envp[]) +{ +#ifndef NDEBUG + fwprintf(stderr, L"######### im in ur IDE, compiling ur c0des ########\n"); +#endif + + int pathIndex = -1; + for (int i = 0; envp[i]; ++i) + if (!wcsncmp(envp[i], L"PATH=", 5)) { + pathIndex = i; + break; + } + + if (pathIndex == -1) { + fwprintf(stderr, L"Couldn't find PATH environment variable!\n"); + return -1; + } + + wchar_t* vcbin = wcsstr(envp[pathIndex], L"WebKitTools\\vcbin"); + if (!vcbin) { + fwprintf(stderr, L"Couldn't find WebKitTools\\vcbin in PATH!\n"); + return -1; + } + + wchar_t saved = *vcbin; + *vcbin = 0; + + wchar_t* afterLeadingSemiColon = wcsrchr(envp[pathIndex], ';'); + if (!afterLeadingSemiColon) + afterLeadingSemiColon = envp[pathIndex] + 5; // +5 for the length of "PATH=" + else + afterLeadingSemiColon++; + + *vcbin = saved; + + size_t pathLength = wcslen(envp[pathIndex]); + + wchar_t* trailingSemiColon = wcschr(vcbin, ';'); + if (!trailingSemiColon) + trailingSemiColon = envp[pathIndex] + pathLength; + + int vcbinLength = trailingSemiColon - afterLeadingSemiColon; + + size_t newPathLength = pathLength - vcbinLength; + + wchar_t* newPath = new wchar_t[newPathLength + 1]; + + // Copy everything before the vcbin path... + wchar_t* d = newPath; + wchar_t* s = envp[pathIndex]; + while (s < afterLeadingSemiColon) + *d++ = *s++; + + // Copy everything after the vcbin path... + s = trailingSemiColon; + while (*d++ = *s++); + + envp[pathIndex] = newPath; + +#ifndef NDEBUG + fwprintf(stderr, L"New path: %s\n", envp[pathIndex]); +#endif + + wchar_t** newArgv = new wchar_t*[argc + 1]; + for (int i = 0; i < argc; ++i) { + size_t length = wcslen(argv[i]); + newArgv[i] = new wchar_t[length + 3]; + *newArgv[i] = '\"'; + wcscpy_s(newArgv[i] + 1, length + 2, argv[i]); + *(newArgv[i] + 1 + length) = '\"'; + *(newArgv[i] + 2 + length) = 0; + } + newArgv[argc] = 0; + + return _wspawnvpe(_P_WAIT, L"midl", newArgv, envp); +} diff --git a/WebKitTools/MIDLWrapper/MIDLWrapper.sln b/WebKitTools/MIDLWrapper/MIDLWrapper.sln new file mode 100644 index 0000000..e0eb2e9 --- /dev/null +++ b/WebKitTools/MIDLWrapper/MIDLWrapper.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 9.00 +# Visual Studio 2005 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "MIDLWrapper", "MIDLWrapper.vcproj", "{CBE6BA0B-1A76-4936-BF54-7EB84E1B0F21}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CBE6BA0B-1A76-4936-BF54-7EB84E1B0F21}.Debug|Win32.ActiveCfg = Debug|Win32 + {CBE6BA0B-1A76-4936-BF54-7EB84E1B0F21}.Debug|Win32.Build.0 = Debug|Win32 + {CBE6BA0B-1A76-4936-BF54-7EB84E1B0F21}.Release|Win32.ActiveCfg = Release|Win32 + {CBE6BA0B-1A76-4936-BF54-7EB84E1B0F21}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/WebKitTools/MIDLWrapper/MIDLWrapper.vcproj b/WebKitTools/MIDLWrapper/MIDLWrapper.vcproj new file mode 100644 index 0000000..d9ab9f9 --- /dev/null +++ b/WebKitTools/MIDLWrapper/MIDLWrapper.vcproj @@ -0,0 +1,199 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/WebKitTools/Scripts/num-cpus b/WebKitTools/Scripts/num-cpus old mode 100644 new mode 100755 diff --git a/WebKitTools/Scripts/parallelcl b/WebKitTools/Scripts/parallelcl new file mode 100755 index 0000000..532079f --- /dev/null +++ b/WebKitTools/Scripts/parallelcl @@ -0,0 +1,224 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use File::Basename; +use File::Spec; +use File::Temp; +use POSIX; + +sub makeJob(\@$); +sub forkAndCompileFiles(\@$); +sub Exec($); +sub waitForChild(\@); +sub cleanup(\@); + +my $debug = 0; + +chomp(my $clexe = `cygpath -u '$ENV{'VS80COMNTOOLS'}/../../VC/bin/cl.exe'`); + +if ($debug) { + print STDERR "Received " . @ARGV . " arguments:\n"; + foreach my $arg (@ARGV) { + print STDERR "$arg\n"; + } +} + +my $commandFile; +foreach my $arg (@ARGV) { + if ($arg =~ /^[\/-](E|EP|P)$/) { + print STDERR "The invoking process wants preprocessed source, so let's hand off this whole command to the real cl.exe\n" if $debug; + Exec("\"$clexe\" \"" . join('" "', @ARGV) . "\""); + } elsif ($arg =~ /^@(.*)$/) { + chomp($commandFile = `cygpath -u '$1'`); + } +} + +die "No command file specified!" unless $commandFile; +die "Couldn't find $commandFile!" unless -f $commandFile; + +my @sources; + +open(COMMAND, '<:raw:encoding(UTF16-LE):crlf:utf8', $commandFile) or die "Couldn't open $commandFile!"; + +# The first line of the command file contains all the options to cl.exe plus the first (possibly quoted) filename +my $firstLine = ; +$firstLine =~ s/\r?\n$//; + +# To find the start of the first filename, look for either the last space on the line. +# If the filename is quoted, the last character on the line will be a quote, so look for the quote before that. +my $firstFileIndex; +print STDERR "Last character of first line = '" . substr($firstLine, -1, 1) . "'\n" if $debug; +if (substr($firstLine, -1, 1) eq '"') { + print STDERR "First file is quoted\n" if $debug; + $firstFileIndex = rindex($firstLine, '"', length($firstLine) - 2); +} else { + print STDERR "First file is NOT quoted\n" if $debug; + $firstFileIndex = rindex($firstLine, ' ') + 1; +} + +my $options = substr($firstLine, 0, $firstFileIndex) . join(' ', @ARGV[1 .. $#ARGV]); +my $possibleFirstFile = substr($firstLine, $firstFileIndex); +if ($possibleFirstFile =~ /\.(cpp|c)/) { + push(@sources, $possibleFirstFile); +} else { + $options .= " $possibleFirstFile"; +} + +print STDERR "######## Found options $options ##########\n" if $debug; +print STDERR "####### Found first source file $sources[0] ########\n" if @sources && $debug; + +# The rest of the lines of the command file just contain source files, one per line +while (my $source = ) { + chomp($source); + $source =~ s/^\s+//; + $source =~ s/\s+$//; + push(@sources, $source) if length($source); +} +close(COMMAND); + +my $numSources = @sources; +exit unless $numSources > 0; + +my $numJobs; +if ($options =~ s/-j\s*([0-9]+)//) { + $numJobs = $1; +} else { + chomp($numJobs = `num-cpus`); +} + +print STDERR "\n\n####### RUNNING AT MOST $numJobs PARALLEL INSTANCES OF cl.exe ###########\n\n";# if $debug; + +# Magic determination of job size +# The hope is that by splitting the source files up into 2*$numJobs pieces, we +# won't suffer too much if one job finishes much more quickly than another. +# However, we don't want to split it up too much due to cl.exe overhead, so set +# the minimum job size to 5. +my $jobSize = POSIX::ceil($numSources / (2 * $numJobs)); +$jobSize = $jobSize < 5 ? 5 : $jobSize; + +print STDERR "######## jobSize = $jobSize ##########\n" if $debug; + +# Sort the source files randomly so that we don't end up with big clumps of large files (aka SVG) +sub fisher_yates_shuffle(\@) +{ + my ($array) = @_; + for (my $i = @{$array}; --$i; ) { + my $j = int(rand($i+1)); + next if $i == $j; + @{$array}[$i,$j] = @{$array}[$j,$i]; + } +} + +fisher_yates_shuffle(@sources); # permutes @array in place + +my @children; +my @tmpFiles; +my $status = 0; +while (@sources) { + while (@sources && @children < $numJobs) { + my $pid; + my $tmpFile; + my $job = makeJob(@sources, $jobSize); + ($pid, $tmpFile) = forkAndCompileFiles(@{$job}, $options); + + print STDERR "####### Spawned child with PID $pid and tmpFile $tmpFile ##########\n" if $debug; + push(@children, $pid); + push(@tmpFiles, $tmpFile); + } + + $status |= waitForChild(@children); +} + +while (@children) { + $status |= waitForChild(@children); +} +cleanup(@tmpFiles); + +exit WEXITSTATUS($status); + + +sub makeJob(\@$) +{ + my ($files, $jobSize) = @_; + + my @job; + if (@{$files} > ($jobSize * 1.5)) { + @job = splice(@{$files}, -$jobSize); + } else { + # Compile all the remaining files in this job to avoid having a small job later + @job = splice(@{$files}); + } + + return \@job; +} + +sub forkAndCompileFiles(\@$) +{ + print STDERR "######## forkAndCompileFiles()\n" if $debug; + my ($files, $options) = @_; + + if ($debug) { + foreach my $file (@{$files}) { + print STDERR "######## $file\n"; + } + } + + my (undef, $tmpFile) = File::Temp::tempfile('clcommandXXXXX', DIR => File::Spec->tmpdir, OPEN => 0); + + my $pid = fork(); + die "Fork failed" unless defined($pid); + + unless ($pid) { + # Child process + open(TMP, '>:raw:encoding(UTF16-LE):crlf:utf8', $tmpFile) or die "Couldn't open $tmpFile"; + print TMP "$options\n"; + foreach my $file (@{$files}) { + print TMP "$file\n"; + } + close(TMP); + + chomp(my $winTmpFile = `cygpath -m $tmpFile`); + Exec "\"$clexe\" \@\"$winTmpFile\""; + } else { + return ($pid, $tmpFile); + } +} + +sub Exec($) +{ + my ($command) = @_; + + print STDERR "Exec($command)\n" if $debug; + + exec($command); +} + +sub waitForChild(\@) +{ + my ($children) = @_; + + return unless @{$children}; + + my $deceased = wait(); + my $status = $?; + print STDERR "######## Child with PID $deceased finished ###########\n" if $debug; + for (my $i = 0; $i < @{$children}; $i++) { + if ($children->[$i] == $deceased) { + splice(@{$children}, $i, 1); + last; + } + } + + return $status; +} + +sub cleanup(\@) +{ + my ($tmpFiles) = @_; + + foreach my $file (@{$tmpFiles}) { + unlink $file; + } +} diff --git a/WebKitTools/Scripts/pdevenv b/WebKitTools/Scripts/pdevenv new file mode 100755 index 0000000..66a4ece --- /dev/null +++ b/WebKitTools/Scripts/pdevenv @@ -0,0 +1,24 @@ +#!/usr/bin/perl -w + +use strict; +use warnings; + +use File::Temp qw/tempfile/; +use FindBin; + +my ($fh, $path) = tempfile(UNLINK => 0, SUFFIX => '.cmd') or die; + +chomp(my $vcBin = `cygpath -w "$FindBin::Bin/../vcbin"`); + +print $fh "\@echo off\n\n"; +print $fh "call \"\%VS80COMNTOOLS\%\\vsvars32.bat\"\n\n"; +print $fh "set PATH=$vcBin;\%PATH\%\n\n"; +print $fh "devenv /useenv\n"; + +close $fh; + +chmod 0755, $path; + +chomp($path = `cygpath -w -s '$path'`); + +exec("cmd /c \"call $path\""); diff --git a/WebKitTools/vcbin/cl.exe b/WebKitTools/vcbin/cl.exe new file mode 100755 index 0000000..2ec78c9 Binary files /dev/null and b/WebKitTools/vcbin/cl.exe differ diff --git a/WebKitTools/vcbin/midl.exe b/WebKitTools/vcbin/midl.exe new file mode 100755 index 0000000..55ecd1d Binary files /dev/null and b/WebKitTools/vcbin/midl.exe differ