Skip to content

Instantly share code, notes, and snippets.

@TheSnoozer

TheSnoozer/measure_performance.py

Last active August 6, 2020 19:53
Show Gist options
  • Save TheSnoozer/7891eb9412c9584a052f00650d9c516b to your computer and use it in GitHub Desktop.
Save TheSnoozer/7891eb9412c9584a052f00650d9c516b to your computer and use it in GitHub Desktop.
Download ZIP
Simple script to measure the execution time of git-commit-id/maven-git-commit-id-plugin
Raw
measure_performance.py
# python3.6 -m pip install GitPython
import os
import re
import git
import time
import tempfile
import subprocess
from git import RemoteProgress
TEST_DIR = '/tmp/testing'
GIT_URLS = [
# [email protected]:torvalds/linux.git
'[email protected]:pockethub/PocketHub.git',
# '[email protected]:cstamas/nexus-core-ng.git',
'[email protected]:apache/maven.git',
'[email protected]:apache/wicket.git',
'[email protected]:netty/netty.git',
'[email protected]:tensorflow/tensorflow.git',
'[email protected]:angular/angular.js.git',
'[email protected]:twbs/bootstrap.git',
# '[email protected]:kubernetes/kubernetes.git',
# '[email protected]:apache/spark.git',
# '[email protected]:microsoft/vscode.git',
# '[email protected]:rust-lang/rust.git',
]
def mean(numbers):
return float(sum(numbers)) / max(len(numbers), 1)
class BenchmarkResult:
def __init__(
self, runtime_old_version_with_native, runtime_old_version_without_native,
runtime_new_version_with_native, runtime_new_version_without_native):
self.runtime_old_version_with_native = runtime_old_version_with_native
self.runtime_old_version_without_native = runtime_old_version_without_native
self.runtime_new_version_with_native = runtime_new_version_with_native
self.runtime_new_version_without_native = runtime_new_version_without_native
print(self.__dict__)
def __add__(self, other):
if isinstance(other, self.__class__):
return BenchmarkResult(
self.runtime_old_version_with_native + other.runtime_old_version_with_native,
self.runtime_old_version_without_native + other.runtime_old_version_without_native,
self.runtime_new_version_with_native + other.runtime_new_version_with_native,
self.runtime_new_version_without_native + other.runtime_new_version_without_native)
else:
raise TypeError(f"Can only perform addition on {self.__class__} objects!")
def __radd__(self, other):
""" http://www.marinamele.com/2014/04/modifying-add-method-of-python-class.html """
if other == 0:
return self
else:
return self.__add__(other)
def devide_by(self, num):
return BenchmarkResult(
self.runtime_old_version_with_native / max(num, 1),
self.runtime_old_version_without_native / max(num, 1),
self.runtime_new_version_with_native / max(num, 1),
self.runtime_new_version_without_native / max(num, 1))
def to_details_text(self, old_version, new_version):
return f"""
"{old_version}","True","{self.runtime_old_version_with_native}"
"{new_version}","True","{self.runtime_new_version_with_native}"
"{old_version}","False","{self.runtime_old_version_without_native}"
"{new_version}","False","{self.runtime_new_version_without_native}"
"""
class AveragedBenchmarkResult:
def __init__(self, repository_url, commit_count, benchmark_results):
self.repository_url = repository_url
self.commit_count = commit_count
self.benchmark_results = benchmark_results
self.combined_mean = sum(benchmark_results).devide_by(len(benchmark_results))
def to_html(self):
return f"""
<tr>
<td>
<p>{self.repository_url}</p>
</td>
<td><p>{self.commit_count}</p></td>
<td><p>{self.combined_mean.runtime_old_version_with_native:.4f}</p></td>
<td><p>{self.combined_mean.runtime_new_version_with_native:.4f}</p></td>
<td><p>{self.combined_mean.runtime_old_version_without_native:.4f}</p></td>
<td><p>{self.combined_mean.runtime_new_version_without_native:.4f}</p></td>
</tr>"""
def to_details_text(self, old_version, new_version):
return f"""
=== Run test for {self.repository_url} (commit count: {self.commit_count})
{''.join(benchmark_result.to_details_text(old_version, new_version)
for benchmark_result in self.benchmark_results)}
"""
class CustomProgress(RemoteProgress):
def update(self, op_code, cur_count, max_count=None, message=''):
if message:
percent = cur_count / max_count
print("{0:50}\t{1:.2%}".format(message, percent), end='\r')
if op_code & RemoteProgress.END == RemoteProgress.END:
print('')
def git_url_to_path(base_dir, git_url):
repo_name = re.findall('/(.+?)\.git', git_url)[0]
repo_path = os.path.join(base_dir, repo_name)
return repo_path
def setup_testing_repos():
os.makedirs(TEST_DIR, exist_ok=True)
for git_url in GIT_URLS:
repo_path = git_url_to_path(TEST_DIR, git_url)
if not os.path.isdir(repo_path):
print('Clone {0} into {1}'.format(git_url, repo_path))
git.Repo.clone_from(git_url, repo_path, progress=CustomProgress())
else:
print('Nothing to-do for {0}'.format(git_url))
def setup_maven_benchmark_pom_without_children(tmp_dir_name, extra_config_for_git_plugin):
with open(os.path.join(tmp_dir_name, "pom.xml"), 'w') as mvn_pom:
mvn_pom.write(f"""<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>9</version>
</parent>
<artifactId>naive-performance-test</artifactId>
<version>0.0.3-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<git-commit-id-version>3.0.0-SNAPSHOT</git-commit-id-version>
<git-use-native>false</git-use-native>
</properties>
<build>
<plugins>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>${{git-commit-id-version}}</version>
<executions>
<execution>
<id>get-the-git-infos</id>
<goals>
<goal>revision</goal>
</goals>
<phase>initialize</phase>
</execution>
</executions>
<configuration>
<prefix>git</prefix>
<verbose>true</verbose>
<useNativeGit>${{git-use-native}}</useNativeGit>
<skipPoms>false</skipPoms>
<dotGitDirectory>${{project.basedir}}/.git</dotGitDirectory>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<evaluateOnCommit>HEAD</evaluateOnCommit>
<generateGitPropertiesFilename>${{project.build.outputDirectory}}/git.properties</generateGitPropertiesFilename>
{extra_config_for_git_plugin}
</configuration>
</plugin>
</plugins>
</build>
</project>""")
def setup_maven_benchmark_pom_with_children(tmp_dir_name, extra_config_for_git_plugin, number_of_child_projects):
with open(os.path.join(tmp_dir_name, "pom.xml"), 'w') as root_pom:
root_pom.write(f"""<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>9</version>
</parent>
<groupId>pl.project13.maven</groupId>
<artifactId>naive-performance-test-parent</artifactId>
<version>0.0.3-SNAPSHOT</version>
<packaging>pom</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.target>1.8</java.target>
<git-commit-id-version>3.0.0-SNAPSHOT</git-commit-id-version>
</properties>
<build>
<plugins>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>${{git-commit-id-version}}</version>
<executions>
<execution>
<id>get-the-git-infos</id>
<goals>
<goal>revision</goal>
</goals>
<phase>initialize</phase>
</execution>
</executions>
<configuration>
<prefix>git</prefix>
<verbose>true</verbose>
<useNativeGit>${{git-use-native}}</useNativeGit>
<skipPoms>false</skipPoms>
<dotGitDirectory>${{project.basedir}}/.git</dotGitDirectory>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<evaluateOnCommit>HEAD</evaluateOnCommit>
<generateGitPropertiesFilename>${{project.build.outputDirectory}}/git.properties</generateGitPropertiesFilename>
{extra_config_for_git_plugin}
</configuration>
</plugin>
</plugins>
</build>
<modules>
{(os.linesep + " ").join(
[f"<module>child_{project_num}</module>" for project_num in range(number_of_child_projects)])
}
</modules>
</project>""")
for project_num in range(number_of_child_projects):
child_dir = os.path.join(tmp_dir_name, f"child_{project_num}")
os.path.isdir(child_dir) or os.mkdir(child_dir)
with open(os.path.join(child_dir, "pom.xml"), 'w') as mvn_pom:
mvn_pom.write(f"""<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>pl.project13.maven</groupId>
<artifactId>naive-performance-test-parent</artifactId>
<version>0.0.3-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>naive-performance-test-{project_num}</artifactId>
<version>0.0.3-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<git-commit-id-version>3.0.0-SNAPSHOT</git-commit-id-version>
<git-use-native>false</git-use-native>
</properties>
<build>
<plugins>
<plugin>
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>${{git-commit-id-version}}</version>
<executions>
<execution>
<id>get-the-git-infos</id>
<goals>
<goal>revision</goal>
</goals>
<phase>initialize</phase>
</execution>
</executions>
<configuration>
<prefix>git</prefix>
<verbose>true</verbose>
<useNativeGit>${{git-use-native}}</useNativeGit>
<skipPoms>false</skipPoms>
<dotGitDirectory>${{project.basedir}}/../.git</dotGitDirectory>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
<evaluateOnCommit>HEAD</evaluateOnCommit>
<generateGitPropertiesFilename>${{project.build.outputDirectory}}/git.properties</generateGitPropertiesFilename>
{extra_config_for_git_plugin}
</configuration>
</plugin>
</plugins>
</build>
</project>""")
def dump_final_results(benchmark_file, test_name, old_version, new_version, averaged_benchmark_results):
benchmark_file.write(f"""
<h2>{test_name}</h2>
<table border="0" cellspacing="0" cellpadding="0">
<colgroup>
<col width="257" />
<col width="165" />
<col width="182" />
<col width="208" />
<col width="99" />
<col width="208" />
</colgroup>
<tbody>
<tr />
<tr>
<td rowspan="2"><p>Repository</p></td>
<td rowspan="2"><p>Commit Count</p></td>
<td colspan="2"><p>use-native=True</p></td>
<td colspan="2"><p>use-native=False</p></td>
</tr>
<tr>
<td><p>{old_version}</p></td>
<td><p>{new_version}</p></td>
<td><p>{old_version}</p></td>
<td><p>{new_version}</p></td>
</tr>{''.join(averaged_benchmark_result.to_html()
for averaged_benchmark_result in averaged_benchmark_results)}
</tbody>
</table>
<details>
<summary>Click to view raw data</summary>
```{''.join(averaged_benchmark_result.to_details_text(old_version, new_version)
for averaged_benchmark_result in averaged_benchmark_results)}
```
</details>
""")
def run_process(args, cwd):
# print("start process: " + str(' '.join(args)))
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise ValueError('Process failed with exit-code {0}.\nOutput: {1}'.format(proc.returncode, stdout))
return stdout
def run_benchmark_for_repo(git_url, old_version, new_version, extra_config_for_git_plugin, number_of_child_projects):
repo_path = git_url_to_path(TEST_DIR, git_url)
actual_git_repo = os.path.join(repo_path, '.git')
commit_count = run_process(['git', 'rev-list', '--all', '--count'], cwd=repo_path)
commit_count = re.search(r'\d+', str(commit_count)).group()
print(f"=== Run test for {repo_path} (commit count: {commit_count})")
if number_of_child_projects == 0:
setup_maven_benchmark_pom_without_children(repo_path, extra_config_for_git_plugin)
else:
setup_maven_benchmark_pom_with_children(repo_path, extra_config_for_git_plugin, number_of_child_projects)
benchmark_results = []
max_attempts = 10
for attempt in range(1, max_attempts + 1):
time.sleep(1)
runtime_old_version_with_native = 0
runtime_new_version_with_native = 0
runtime_old_version_without_native = 0
runtime_new_version_without_native = 0
for use_native_git in [True, False]:
for git_commit_id_plugin_version in [old_version, new_version]:
print(f"Launching {attempt} / {max_attempts} for {git_commit_id_plugin_version},{use_native_git}",
end='\r', flush=True)
start = time.time()
run_process(
['mvn', 'clean',
'pl.project13.maven:git-commit-id-plugin:' + str(git_commit_id_plugin_version) + ':revision',
'-Dgit-use-native=' + str(use_native_git).lower()],
cwd=repo_path)
total_time = time.time() - start
# craft benchmark result
if use_native_git:
if git_commit_id_plugin_version == old_version:
runtime_old_version_with_native = total_time
else:
runtime_new_version_with_native = total_time
else:
if git_commit_id_plugin_version == old_version:
runtime_old_version_without_native = total_time
else:
runtime_new_version_without_native = total_time
benchmark_result = BenchmarkResult(
runtime_old_version_with_native=runtime_old_version_with_native,
runtime_new_version_with_native=runtime_new_version_with_native,
runtime_old_version_without_native=runtime_old_version_without_native,
runtime_new_version_without_native=runtime_new_version_without_native)
benchmark_results.append(benchmark_result)
# cleanup
run_process(['git', 'clean', '-fd'], cwd=repo_path)
return AveragedBenchmarkResult(
repository_url=git_url, commit_count=commit_count, benchmark_results=benchmark_results)
# ===================================================================================================
# CONFIG
version_compare = ['4.0.1', '4.0.2']
extra_config_for_git_plugin = """<runOnlyOnce>true</runOnlyOnce>"""
if __name__ == "__main__":
if len(version_compare) != 2:
raise ValueError("Can only compare two versions at a time")
setup_testing_repos()
old_version, new_version = version_compare
with tempfile.NamedTemporaryFile(prefix='benchmark_', suffix='.html', mode="a", delete=False) as benchmark_file:
print(f"Report will be available under {benchmark_file.name}....")
benchmark_file.write(f"<h1>Compare version {old_version} with {new_version}</h1>")
for number_of_child_projects in [0, 10]:
for test_run_config in [
(
"all properties",
""
),
(
"includeOnlyProperty = ^git.commit.id$",
"""
<includeOnlyProperties>
<includeOnlyProperty>^git.commit.id$</includeOnlyProperty>
</includeOnlyProperties>
"""
),
(
"excludeProperties = ^git.local.branch.*$",
"""
<excludeProperties>
<excludeProperty>^git.local.branch.*$</excludeProperty>
</excludeProperties>
"""
),
]:
sub_test_name, sub_text_extra_config = test_run_config
extra_config_for_git_plugin = extra_config_for_git_plugin + sub_text_extra_config
averaged_benchmark_results = []
if number_of_child_projects == 0:
test_name = f"Run with one project and {sub_test_name}"
else:
test_name = f"Run with {number_of_child_projects} projects and {sub_test_name}"
print(f"=== Running tests for '{test_name}', compare versions {version_compare}")
time.sleep(5)
for git_url in GIT_URLS:
averaged_benchmark_result = run_benchmark_for_repo(
git_url, old_version, new_version, extra_config_for_git_plugin, number_of_child_projects)
averaged_benchmark_results.append(averaged_benchmark_result)
dump_final_results(benchmark_file, test_name, old_version, new_version, averaged_benchmark_results)
benchmark_file.flush()
print(f"Final results are available in {benchmark_file.name}")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment