wine/gecko
0
0
Fork 0
mirror of https://gitlab.winehq.org/wine/wine-gecko.git synced 2024-09-13 09:24:08 -07:00
Code Issues Packages Projects Releases Wiki Activity

Bug 1225364 - remove non-gecko parts of mozharness; r=jlund

Browse Source
This commit is contained in:
Hal Wine 2016-01-14 15:22:21 -08:00
parent 0119231cbd
commit 54b46eb2fb
11 changed files with 0 additions and 4544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
testing/mozharness/configs/b2g_bumper/master.py
View File

@ -1,100 +0,0 @@
#!/usr/bin/env python
config = {
"exes": {
# Get around the https warnings
"hg": ['/usr/local/bin/hg', "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
"hgtool.py": ["/usr/local/bin/hgtool.py"],
"gittool.py": ["/usr/local/bin/gittool.py"],
},
'gecko_pull_url': 'https://hg.mozilla.org/integration/b2g-inbound',
'gecko_push_url': 'ssh://hg.mozilla.org/integration/b2g-inbound',
'gecko_local_dir': 'b2g-inbound',
'git_ref_cache': '/builds/b2g_bumper/git_ref_cache.json',
'manifests_repo': 'https://git.mozilla.org/b2g/b2g-manifest.git',
'manifests_revision': 'origin/master',
'hg_user': 'B2G Bumper Bot <release+b2gbumper@mozilla.com>',
"ssh_key": "~/.ssh/ffxbld_rsa",
"ssh_user": "ffxbld",
'hgtool_base_bundle_urls': ['https://ftp-ssl.mozilla.org/pub/mozilla.org/firefox/bundles'],
'gaia_repo_url': 'https://hg.mozilla.org/integration/gaia-central',
'gaia_revision_file': 'b2g/config/gaia.json',
'gaia_max_revisions': 1,
# Which git branch this hg repo corresponds to
'gaia_git_branch': 'master',
'gaia_git_repo': 'https://git.mozilla.org/releases/gaia.git',
'gaia_mapper_project': 'gaia',
'mapper_url': 'http://cruncher.build.mozilla.org/mapper/{project}/{vcs}/{rev}',
'devices': {
'dolphin': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-l': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-kk': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-jb': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-ics': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
'manifest_file': 'emulator.xml',
},
# Equivalent to emulator-ics - see bug 916134
# Remove once the above bug resolved
'emulator': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
'manifest_file': 'emulator.xml',
},
'flame-kk': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'nexus-4': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'nexus-4-kk': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'nexus-5-l': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'aries': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
'manifest_file': 'shinano.xml',
},
},
'repo_remote_mappings': {
'https://android.googlesource.com/': 'https://git.mozilla.org/external/aosp',
'git://codeaurora.org/': 'https://git.mozilla.org/external/caf',
'git://github.com/mozilla-b2g/': 'https://git.mozilla.org/b2g',
'git://github.com/mozilla/': 'https://git.mozilla.org/b2g',
'https://git.mozilla.org/releases': 'https://git.mozilla.org/releases',
'http://android.git.linaro.org/git-ro/': 'https://git.mozilla.org/external/linaro',
'http://sprdsource.spreadtrum.com:8085/b2g/android': 'https://git.mozilla.org/external/sprd-aosp',
'git://github.com/apitrace/': 'https://git.mozilla.org/external/apitrace',
'git://github.com/t2m-foxfone/': 'https://git.mozilla.org/external/t2m-foxfone',
# Some mappings to ourself, we want to leave these as-is!
'https://git.mozilla.org/external/aosp': 'https://git.mozilla.org/external/aosp',
'https://git.mozilla.org/external/caf': 'https://git.mozilla.org/external/caf',
'https://git.mozilla.org/b2g': 'https://git.mozilla.org/b2g',
'https://git.mozilla.org/external/apitrace': 'https://git.mozilla.org/external/apitrace',
'https://git.mozilla.org/external/t2m-foxfone': 'https://git.mozilla.org/external/t2m-foxfone',
},
}

87
testing/mozharness/configs/b2g_bumper/v2.1s.py
View File

@ -1,87 +0,0 @@
#!/usr/bin/env python
config = {
"exes": {
# Get around the https warnings
"hg": ['/usr/local/bin/hg', "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
"hgtool.py": ["/usr/local/bin/hgtool.py"],
"gittool.py": ["/usr/local/bin/gittool.py"],
},
'gecko_pull_url': 'https://hg.mozilla.org/releases/mozilla-b2g34_v2_1s/',
'gecko_push_url': 'ssh://hg.mozilla.org/releases/mozilla-b2g34_v2_1s/',
'gecko_local_dir': 'mozilla-b2g34_v2_1s',
'git_ref_cache': '/builds/b2g_bumper/git_ref_cache.json',
'manifests_repo': 'https://git.mozilla.org/b2g/b2g-manifest.git',
'manifests_revision': 'origin/v2.1s',
'hg_user': 'B2G Bumper Bot <release+b2gbumper@mozilla.com>',
"ssh_key": "~/.ssh/ffxbld_rsa",
"ssh_user": "ffxbld",
'hgtool_base_bundle_urls': ['https://ftp-ssl.mozilla.org/pub/mozilla.org/firefox/bundles'],
'gaia_repo_url': 'https://hg.mozilla.org/integration/gaia-2_1s',
'gaia_revision_file': 'b2g/config/gaia.json',
'gaia_max_revisions': 5,
# Which git branch this hg repo corresponds to
'gaia_git_branch': 'v2.1s',
'gaia_git_repo': 'https://git.mozilla.org/releases/gaia.git',
'gaia_mapper_project': 'gaia',
'mapper_url': 'http://cruncher.build.mozilla.org/mapper/{project}/{vcs}/{rev}',
'devices': {
'dolphin': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'dolphin-512': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-kk': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-jb': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-ics': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
'manifest_file': 'emulator.xml',
},
# Equivalent to emulator-ics - see bug 916134
# Remove once the above bug resolved
'emulator': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
'manifest_file': 'emulator.xml',
},
'flame-kk': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'nexus-4': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
},
'repo_remote_mappings': {
'https://android.googlesource.com/': 'https://git.mozilla.org/external/aosp',
'git://codeaurora.org/': 'https://git.mozilla.org/external/caf',
'git://github.com/mozilla-b2g/': 'https://git.mozilla.org/b2g',
'git://github.com/mozilla/': 'https://git.mozilla.org/b2g',
'https://git.mozilla.org/releases': 'https://git.mozilla.org/releases',
'http://android.git.linaro.org/git-ro/': 'https://git.mozilla.org/external/linaro',
'http://sprdsource.spreadtrum.com:8085/b2g/android': 'https://git.mozilla.org/external/sprd-aosp',
'git://github.com/apitrace/': 'https://git.mozilla.org/external/apitrace',
'git://github.com/t2m-foxfone/': 'https://git.mozilla.org/external/t2m-foxfone',
# Some mappings to ourself, we want to leave these as-is!
'https://git.mozilla.org/external/aosp': 'https://git.mozilla.org/external/aosp',
'https://git.mozilla.org/external/caf': 'https://git.mozilla.org/external/caf',
'https://git.mozilla.org/b2g': 'https://git.mozilla.org/b2g',
'https://git.mozilla.org/external/apitrace': 'https://git.mozilla.org/external/apitrace',
'https://git.mozilla.org/external/t2m-foxfone': 'https://git.mozilla.org/external/t2m-foxfone',
},
}

91
testing/mozharness/configs/b2g_bumper/v2.2.py
View File

@ -1,91 +0,0 @@
#!/usr/bin/env python
config = {
"exes": {
# Get around the https warnings
"hg": ['/usr/local/bin/hg', "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
"hgtool.py": ["/usr/local/bin/hgtool.py"],
"gittool.py": ["/usr/local/bin/gittool.py"],
},
'gecko_pull_url': 'https://hg.mozilla.org/releases/mozilla-b2g37_v2_2/',
'gecko_push_url': 'ssh://hg.mozilla.org/releases/mozilla-b2g37_v2_2/',
'gecko_local_dir': 'mozilla-b2g37_v2_2',
'git_ref_cache': '/builds/b2g_bumper/git_ref_cache.json',
'manifests_repo': 'https://git.mozilla.org/b2g/b2g-manifest.git',
'manifests_revision': 'origin/v2.2',
'hg_user': 'B2G Bumper Bot <release+b2gbumper@mozilla.com>',
"ssh_key": "~/.ssh/ffxbld_rsa",
"ssh_user": "ffxbld",
'hgtool_base_bundle_urls': ['https://ftp-ssl.mozilla.org/pub/mozilla.org/firefox/bundles'],
'gaia_repo_url': 'https://hg.mozilla.org/integration/gaia-2_2',
'gaia_revision_file': 'b2g/config/gaia.json',
'gaia_max_revisions': 5,
# Which git branch this hg repo corresponds to
'gaia_git_branch': 'v2.2',
'gaia_git_repo': 'https://git.mozilla.org/releases/gaia.git',
'gaia_mapper_project': 'gaia',
'mapper_url': 'http://cruncher.build.mozilla.org/mapper/{project}/{vcs}/{rev}',
'devices': {
'dolphin': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-l': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-kk': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-jb': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'emulator-ics': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
'manifest_file': 'emulator.xml',
},
# Equivalent to emulator-ics - see bug 916134
# Remove once the above bug resolved
'emulator': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
'manifest_file': 'emulator.xml',
},
'flame-kk': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'nexus-4': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
'nexus-5-l': {
'ignore_projects': ['gecko'],
'ignore_groups': ['darwin'],
},
},
'repo_remote_mappings': {
'https://android.googlesource.com/': 'https://git.mozilla.org/external/aosp',
'git://codeaurora.org/': 'https://git.mozilla.org/external/caf',
'git://github.com/mozilla-b2g/': 'https://git.mozilla.org/b2g',
'git://github.com/mozilla/': 'https://git.mozilla.org/b2g',
'https://git.mozilla.org/releases': 'https://git.mozilla.org/releases',
'http://android.git.linaro.org/git-ro/': 'https://git.mozilla.org/external/linaro',
'http://sprdsource.spreadtrum.com:8085/b2g/android': 'https://git.mozilla.org/external/sprd-aosp',
'git://github.com/apitrace/': 'https://git.mozilla.org/external/apitrace',
'git://github.com/t2m-foxfone/': 'https://git.mozilla.org/external/t2m-foxfone',
# Some mappings to ourself, we want to leave these as-is!
'https://git.mozilla.org/external/aosp': 'https://git.mozilla.org/external/aosp',
'https://git.mozilla.org/external/caf': 'https://git.mozilla.org/external/caf',
'https://git.mozilla.org/b2g': 'https://git.mozilla.org/b2g',
'https://git.mozilla.org/external/apitrace': 'https://git.mozilla.org/external/apitrace',
'https://git.mozilla.org/external/t2m-foxfone': 'https://git.mozilla.org/external/t2m-foxfone',
},
}

789
testing/mozharness/configs/vcs_sync/beagle.py
View File

@ -1,789 +0,0 @@
# This is for gecko-dev, which is a developer-oriented repo with
# release-train and inbound branches.
import os
import socket
hostname = socket.gethostname()
CVS_MANIFEST = """[{
"size": 1301484692,
"digest": "89df462d8d20f54402caaaa4e3c10aa54902a1d7196cdf86b7790b76e62d302ade3102dc3f7da4145dd832e6938b0472370ce6a321e0b3bcf0ad050937bd0e9a",
"algorithm": "sha512",
"filename": "mozilla-cvs-history.tar.bz2"
}]
"""
config = {
"log_name": "beagle",
"log_max_rotate": 99,
"repos": [{
"repo": "https://hg.mozilla.org/users/hwine_mozilla.com/repo-sync-tools",
"vcs": "hg",
}],
"job_name": "beagle",
"conversion_dir": "beagle",
"initial_repo": {
"repo": "https://hg.mozilla.org/mozilla-central",
"revision": "default",
"repo_name": "mozilla-central",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "master",
},
},
},
"backup_dir": "/mnt/netapp/github_sync/aki/%s" % hostname,
"cvs_manifest": CVS_MANIFEST,
"cvs_history_tarball": "/home/pmoore/mozilla-cvs-history.tar.bz2",
"env": {
"PATH": "%(PATH)s:/usr/libexec/git-core",
},
"conversion_repos": [{
"repo": "https://hg.mozilla.org/releases/mozilla-b2g18",
"revision": "default",
"repo_name": "mozilla-b2g18",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g18",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g26_v1_2",
"revision": "default",
"repo_name": "mozilla-b2g26_v1_2",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g26_v1_2",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g26_v1_2f",
"revision": "default",
"repo_name": "mozilla-b2g26_v1_2f",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g26_v1_2f",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g28_v1_3",
"revision": "default",
"repo_name": "mozilla-b2g28_v1_3",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g28_v1_3",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g28_v1_3t",
"revision": "default",
"repo_name": "mozilla-b2g28_v1_3t",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g28_v1_3t",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g30_v1_4",
"revision": "default",
"repo_name": "mozilla-b2g30_v1_4",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g30_v1_4",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g34_v2_1",
"revision": "default",
"repo_name": "mozilla-b2g34_v2_1",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g34_v2_1",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g37_v2_2",
"revision": "default",
"repo_name": "mozilla-b2g37_v2_2",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g37_v2_2",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g37_v2_2r",
"revision": "default",
"repo_name": "mozilla-b2g37_v2_2r",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g37_v2_2r",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g32_v2_0",
"revision": "default",
"repo_name": "mozilla-b2g32_v2_0",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g32_v2_0",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g32_v2_0m",
"revision": "default",
"repo_name": "mozilla-b2g32_v2_0m",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g32_v2_0m",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_1_0_hd",
"revision": "default",
"repo_name": "mozilla-b2g18_v1_1_0_hd",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g18_v1_1_0_hd",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_0_1",
"revision": "default",
"repo_name": "mozilla-b2g18_v1_0_1",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g18_v1_0_1",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_0_0",
"revision": "default",
"repo_name": "mozilla-b2g18_v1_0_0",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g18_v1_0_0",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-aurora",
"revision": "default",
"repo_name": "mozilla-aurora",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "aurora",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-beta",
"revision": "default",
"repo_name": "mozilla-beta",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "beta",
},
"branch_regexes": [
"^GECKO[0-9_]*RELBRANCH$",
"^MOBILE[0-9_]*RELBRANCH$",
],
},
"tag_config": {
"tag_regexes": [
"^(B2G|RELEASE_BASE)_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-release",
"revision": "default",
"repo_name": "mozilla-release",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "release",
},
"branch_regexes": [
"^GECKO[0-9_]*RELBRANCH$",
"^MOBILE[0-9_]*RELBRANCH$",
],
},
"tag_config": {
"tag_regexes": [
"^(B2G|RELEASE_BASE)_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-esr17",
"revision": "default",
"repo_name": "mozilla-esr17",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "esr17",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-esr31",
"revision": "default",
"repo_name": "mozilla-esr31",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "esr31",
},
"branch_regexes": [
"^GECKO[0-9]+esr_[0-9]+_RELBRANCH$",
],
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-esr38",
"revision": "default",
"repo_name": "mozilla-esr38",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "esr38",
},
"branch_regexes": [
"^GECKO[0-9]+esr_[0-9]+_RELBRANCH$",
],
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/integration/mozilla-inbound",
"revision": "default",
"repo_name": "mozilla-inbound",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "inbound",
},
},
"tag_config": {},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/integration/b2g-inbound",
"revision": "default",
"repo_name": "b2g-inbound",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "b2g-inbound",
},
},
"tag_config": {},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}, {
"repo": "https://hg.mozilla.org/integration/fx-team",
"revision": "default",
"repo_name": "fx-team",
"targets": [{
"target_dest": "beagle/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "gitmo-beagle",
}, {
"target_dest": "github-beagle",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "fx-team",
},
},
"tag_config": {},
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "gecko-dev"
},
}],
"remote_targets": {
"github-beagle": {
"repo": "git@github.com:mozilla/gecko-dev.git",
"ssh_key": "~/.ssh/releng-github-id_rsa",
"vcs": "git",
},
"gitmo-beagle": {
"repo": "gitolite3@git.mozilla.org:integration/gecko-dev.git",
"ssh_key": "~/.ssh/vcs-sync_rsa",
"vcs": "git",
},
},
"exes": {
# bug 828140 - shut https warnings up.
# http://kiln.stackexchange.com/questions/2816/mercurial-certificate-warning-certificate-not-verified-web-cacerts
"hg": [os.path.join(os.getcwd(), "build", "venv", "bin", "hg"), "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
"tooltool.py": [
os.path.join(os.getcwd(), "build", "venv", "bin", "python"),
os.path.join(os.getcwd(), "mozharness", "external_tools", "tooltool.py"),
],
},
"virtualenv_modules": [
"bottle==0.11.6",
"dulwich==0.9.0",
"ordereddict==1.1",
"hg-git==0.4.0-moz2",
"mapper==0.1",
"mercurial==2.6.3",
"mozfile==0.9",
"mozinfo==0.5",
"mozprocess==0.11",
"requests==2.8.1",
],
"find_links": [
"http://pypi.pvt.build.mozilla.org/pub",
"http://pypi.pub.build.mozilla.org/pub",
],
"pip_index": False,
"default_notify_from": "developer-services+%s@mozilla.org" % hostname,
"notify_config": [{
"to": "releng-ops-trial@mozilla.com",
"failure_only": False,
"skip_empty_messages": True,
}],
# Disallow sharing, since we want pristine .hg and .git directories.
"vcs_share_base": None,
"hg_share_base": None,
"default_actions": [
'list-repos',
'create-virtualenv',
'update-stage-mirror',
'update-work-mirror',
'publish-to-mapper',
'push',
'combine-mapfiles',
'notify',
],
}

113
testing/mozharness/configs/vcs_sync/build-repos.py
View File

@ -1,113 +0,0 @@
import os
import socket
hostname = socket.gethostname()
build_repos = (
'autoland',
'buildapi',
'buildbot-configs',
'buildbotcustom',
'mozharness',
'opsi-package-sources',
'partner-repacks',
'preproduction',
'puppet',
'puppet-manifests',
'rpm-sources',
'talos',
'tools',
)
conversion_repos = []
remote_targets = {}
for repo in build_repos:
conversion_repos.append({
"repo": "https://hg.mozilla.org/build/%s" % repo,
"repo_name": "build-%s" % repo,
"conversion_dir": "build-%s" % repo,
"targets": [{
"target_dest": "build-%s-github" % repo,
"force_push": True
}],
"vcs": "hg",
"mapper": {
"url": "https://api.pub.build.mozilla.org/mapper",
"project": "build-%s" % repo,
},
"branch_config": {
"branches": {
"default": "master",
},
"branch_regexes": [
"^.*$"
]
},
# Bug 1036819 - build/* repos currently not able to push tags to github
# temporarily disable tags in conversion.
# When bug 1020613 is resolved, this tag_config below can be enabled again.
# "tag_config": {
# "tag_regexes": [
# "^.*$"
# ]
# },
})
remote_targets["build-%s-github" % repo] = {
"repo": "git@github.com:mozilla/build-%s.git" % repo,
"ssh_key": "~/.ssh/releng-github-id_rsa",
"vcs": "git",
}
config = {
"log_name": "build-repos",
"log_max_rotate": 99,
"job_name": "build-repos",
"env": {
"PATH": "%(PATH)s:/usr/libexec/git-core",
},
"conversion_repos": conversion_repos,
"remote_targets": remote_targets,
"virtualenv_modules": [
"dulwich==0.9.0",
"ordereddict==1.1",
"hg-git==0.4.0-moz2",
"mapper==0.1",
"mercurial==2.6.3",
"mozfile==0.9",
"mozinfo==0.5",
"mozprocess==0.11",
"requests==2.8.1",
],
"find_links": [
"http://pypi.pub.build.mozilla.org/pub"
],
"pip_index": False,
"default_notify_from": "developer-services+%s@mozilla.org" % hostname,
"notify_config": [{
"to": "releng-ops-trial@mozilla.com",
"failure_only": False,
"skip_empty_messages": True,
}],
# Disallow sharing, since we want pristine .hg and .git directories.
"vcs_share_base": None,
"hg_share_base": None,
# any hg command line options
"hg_options": (
"--config",
"web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"
),
"default_actions": [
'list-repos',
'create-virtualenv',
'update-stage-mirror',
'update-work-mirror',
'publish-to-mapper',
'push',
'combine-mapfiles',
'notify',
],
}

437
testing/mozharness/configs/vcs_sync/gecko-git.py
View File

@ -1,437 +0,0 @@
# This is for gecko.git, which is a partner-oriented repo with
# B2G release branches + tags.
import os
import socket
hostname = socket.gethostname()
CVS_MANIFEST = """[{
"size": 1301484692,
"digest": "89df462d8d20f54402caaaa4e3c10aa54902a1d7196cdf86b7790b76e62d302ade3102dc3f7da4145dd832e6938b0472370ce6a321e0b3bcf0ad050937bd0e9a",
"algorithm": "sha512",
"filename": "mozilla-cvs-history.tar.bz2"
}]
"""
config = {
"log_name": "gecko-git",
"log_max_rotate": 99,
"repos": [{
"repo": "https://hg.mozilla.org/users/hwine_mozilla.com/repo-sync-tools",
"vcs": "hg",
}],
"job_name": "gecko-git",
"conversion_dir": "gecko-git",
"initial_repo": {
"repo": "https://hg.mozilla.org/mozilla-central",
"revision": "default",
"repo_name": "mozilla-central",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "master",
},
},
},
"backup_dir": "/mnt/netapp/github_sync/aki/%s" % hostname,
"cvs_manifest": CVS_MANIFEST,
"cvs_history_tarball": "/home/pmoore/mozilla-cvs-history.tar.bz2",
"env": {
"PATH": "%(PATH)s:/usr/libexec/git-core",
},
"conversion_repos": [{
"repo": "https://hg.mozilla.org/releases/mozilla-b2g37_v2_2",
"revision": "default",
"repo_name": "mozilla-b2g37_v2_2",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v2.2",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g37_v2_2r",
"revision": "default",
"repo_name": "mozilla-b2g37_v2_2r",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v2.2r",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g34_v2_1",
"revision": "default",
"repo_name": "mozilla-b2g34_v2_1",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v2.1",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g34_v2_1s",
"revision": "default",
"repo_name": "mozilla-b2g34_v2_1s",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v2.1s",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g32_v2_0",
"revision": "default",
"repo_name": "mozilla-b2g32_v2_0",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v2.0",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g32_v2_0m",
"revision": "default",
"repo_name": "mozilla-b2g32_v2_0m",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v2.0m",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g30_v1_4",
"revision": "default",
"repo_name": "mozilla-b2g30_v1_4",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v1.4",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g28_v1_3",
"revision": "default",
"repo_name": "mozilla-b2g28_v1_3",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v1.3",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g28_v1_3t",
"revision": "default",
"repo_name": "mozilla-b2g28_v1_3t",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v1.3t",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g26_v1_2",
"revision": "default",
"repo_name": "mozilla-b2g26_v1_2",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v1.2",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g26_v1_2f",
"revision": "default",
"repo_name": "mozilla-b2g26_v1_2f",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v1.2f",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g18",
"revision": "default",
"repo_name": "mozilla-b2g18",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "gecko-18",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_1_0_hd",
"revision": "default",
"repo_name": "mozilla-b2g18_v1_1_0_hd",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v1.1.0hd",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_0_1",
"revision": "default",
"repo_name": "mozilla-b2g18_v1_0_1",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v1.0.1",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}, {
"repo": "https://hg.mozilla.org/releases/mozilla-b2g18_v1_0_0",
"revision": "default",
"repo_name": "mozilla-b2g18_v1_0_0",
"targets": [{
"target_dest": "gecko-git/.git",
"vcs": "git",
"test_push": True,
}, {
"target_dest": "github-gecko-git",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "v1.0.0",
},
},
"tag_config": {
"tag_regexes": [
"^B2G_",
],
},
}],
"remote_targets": {
"github-gecko-git": {
"repo": "git@github.com:escapewindow/test-gecko-git.git",
"ssh_key": "~/.ssh/escapewindow_github_rsa",
"vcs": "git",
},
},
"exes": {
# bug 828140 - shut https warnings up.
# http://kiln.stackexchange.com/questions/2816/mercurial-certificate-warning-certificate-not-verified-web-cacerts
"hg": [os.path.join(os.getcwd(), "build", "venv", "bin", "hg"), "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
"tooltool.py": [
os.path.join(os.getcwd(), "build", "venv", "bin", "python"),
os.path.join(os.getcwd(), "mozharness", "external_tools", "tooltool.py"),
],
},
"virtualenv_modules": [
"bottle==0.11.6",
"dulwich==0.9.0",
"ordereddict==1.1",
"hg-git==0.4.0-moz2",
"mapper==0.1",
"mercurial==2.6.3",
"mozfile==0.9",
"mozinfo==0.5",
"mozprocess==0.11",
],
"find_links": [
"http://pypi.pvt.build.mozilla.org/pub",
"http://pypi.pub.build.mozilla.org/pub",
],
"pip_index": False,
"upload_config": [{
"ssh_key": "~/.ssh/id_rsa",
"ssh_user": "pmoore",
"remote_host": "github-sync2",
"remote_path": "/home/pmoore/upload/gecko-git-upload",
}],
"default_notify_from": "developer-services+%s@mozilla.org" % hostname,
"notify_config": [{
"to": "releng-ops-trial@mozilla.com",
"failure_only": False,
"skip_empty_messages": True,
}],
# Disallow sharing, since we want pristine .hg and .git directories.
"vcs_share_base": None,
"hg_share_base": None,
}

311
testing/mozharness/configs/vcs_sync/l10n.py
View File

@ -1,311 +0,0 @@
from copy import deepcopy
import socket
hostname = socket.gethostname()
GECKO_BRANCHES = {
'v2.1': 'mozilla-beta',
'v2.2': 'mozilla-central',
}
GECKO_CONFIG_TEMPLATE = {
'mozilla-release': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gecko-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://hg.mozilla.org/releases/mozilla-release/raw-file/default/b2g/locales/all-locales',
'hg_url': 'https://hg.mozilla.org/releases/l10n/mozilla-release/%(locale)s',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gecko/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gecko-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
'mozilla-beta': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gecko-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://hg.mozilla.org/releases/mozilla-beta/raw-file/default/b2g/locales/all-locales',
'hg_url': 'https://hg.mozilla.org/releases/l10n/mozilla-beta/%(locale)s',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gecko/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gecko-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
'mozilla-aurora': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gecko-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://hg.mozilla.org/releases/mozilla-aurora/raw-file/default/b2g/locales/all-locales',
'hg_url': 'https://hg.mozilla.org/releases/l10n/mozilla-aurora/%(locale)s',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gecko/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gecko-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
'mozilla-central': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gecko-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://hg.mozilla.org/mozilla-central/raw-file/default/b2g/locales/all-locales',
'hg_url': 'https://hg.mozilla.org/l10n-central/%(locale)s',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gecko/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gecko-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
}
# Build gecko_config
GECKO_CONFIG = {}
for version, branch in GECKO_BRANCHES.items():
GECKO_CONFIG[branch] = deepcopy(GECKO_CONFIG_TEMPLATE[branch])
GECKO_CONFIG[branch]['git_branch_name'] = version
config = {
"log_name": "l10n",
"log_max_rotate": 99,
"job_name": "l10n",
"env": {
"PATH": "%(PATH)s:/usr/libexec/git-core",
},
"conversion_type": "b2g-l10n",
"combined_mapfile": "l10n-mapfile",
"l10n_config": {
"gecko_config": GECKO_CONFIG,
"gaia_config": {
'v2_1': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gaia-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://raw.github.com/mozilla-b2g/gaia/v2.1/locales/languages_all.json',
'hg_url': 'https://hg.mozilla.org/releases/gaia-l10n/v2_1/%(locale)s',
'git_branch_name': 'v2.1',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gaia/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gaia-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
'v2_0': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gaia-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://raw.github.com/mozilla-b2g/gaia/v2.0/locales/languages_all.json',
'hg_url': 'https://hg.mozilla.org/releases/gaia-l10n/v2_0/%(locale)s',
'git_branch_name': 'v2.0',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gaia/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gaia-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
'v1_4': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gaia-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://raw.github.com/mozilla-b2g/gaia/v1.4/locales/languages_all.json',
'hg_url': 'https://hg.mozilla.org/releases/gaia-l10n/v1_4/%(locale)s',
'git_branch_name': 'v1.4',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gaia/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gaia-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
'v1_3': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gaia-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://raw.github.com/mozilla-b2g/gaia/v1.3/locales/languages_dev.json',
'hg_url': 'https://hg.mozilla.org/releases/gaia-l10n/v1_3/%(locale)s',
'git_branch_name': 'v1.3',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gaia/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gaia-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
'v1_2': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gaia-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://raw.github.com/mozilla-b2g/gaia/v1.2/locales/languages_all.json',
'hg_url': 'https://hg.mozilla.org/releases/gaia-l10n/v1_2/%(locale)s',
'git_branch_name': 'v1.2',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gaia/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gaia-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
'master': {
'generate_git_notes': False, # we can change this when bug 1034725 is resolved
'mapper': {
'project': 'gitmo-gaia-l10n',
'url': 'https://api.pub.build.mozilla.org/mapper'
},
'locales_file_url': 'https://raw.github.com/mozilla-b2g/gaia/master/locales/languages_all.json',
'hg_url': 'https://hg.mozilla.org/gaia-l10n/%(locale)s',
'git_branch_name': 'master',
'targets': [{
'target_dest': 'releases-l10n-%(locale)s-gaia/.git',
'test_push': True,
'vcs': 'git'
}, {
'target_dest': 'gitmo-gaia-l10n-%(locale)s',
}],
'tag_config': {
'tag_regexes': [
'^B2G_',
],
},
},
},
},
"remote_targets": {
"gitmo-gecko-l10n-%(locale)s": {
"repo": 'gitolite3@git.mozilla.org:releases/l10n/%(locale)s/gecko.git',
"ssh_key": "~/.ssh/vcs-sync_rsa",
"vcs": "git",
},
"gitmo-gaia-l10n-%(locale)s": {
"repo": 'gitolite3@git.mozilla.org:releases/l10n/%(locale)s/gaia.git',
"ssh_key": "~/.ssh/vcs-sync_rsa",
"vcs": "git",
},
},
"virtualenv_modules": [
"bottle==0.11.6",
"dulwich==0.9.0",
"ordereddict==1.1",
"hg-git==0.4.0-moz2",
"mapper==0.1",
"mercurial==2.6.3",
"mozfile==0.9",
"mozinfo==0.5",
"mozprocess==0.11",
"requests==2.8.1",
],
"find_links": [
"http://pypi.pub.build.mozilla.org/pub",
],
"pip_index": False,
"default_notify_from": "developer-services+%s@mozilla.org" % hostname,
"notify_config": [{
"to": "releng-ops-trial@mozilla.com",
"failure_only": False,
"skip_empty_messages": True,
}],
# Disallow sharing, since we want pristine .hg and .git directories.
"vcs_share_base": None,
"hg_share_base": None,
# any hg command line options
"hg_options": (
"--config",
"web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"
),
"default_actions": [
'list-repos',
'create-virtualenv',
'update-stage-mirror',
'update-work-mirror',
'publish-to-mapper',
'push',
'combine-mapfiles',
'notify',
],
}

115
testing/mozharness/configs/vcs_sync/project-branches.py
View File

@ -1,115 +0,0 @@
# This is for gecko-projects, which is a low-SLA developer-oriented repo
# with mozilla-central based project branches.
import os
import socket
hostname = socket.gethostname()
# These all need to be under hg.m.o/projects.
# If you need to add a different repo, add it to CONVERSION_REPOS.
PROJECT_BRANCHES = [
# twig projects
"alder",
"ash",
"birch",
"cedar",
"cypress",
"date",
"elm",
"fig",
"gum",
"holly",
"jamun",
"larch",
"maple",
"oak",
"pine",
# Non-twig projects
"build-system",
"graphics",
"ux",
]
# Non-hg.m.o/projects/ repos.
CONVERSION_REPOS = [{
"repo": "https://hg.mozilla.org/services/services-central",
"revision": "default",
"repo_name": "services-central",
"targets": [{
"target_dest": "github-project-branches",
}],
"vcs": "hg",
"branch_config": {
"branches": {
"default": "services",
},
},
"tag_config": {},
}]
config = {
"log_name": "project-branches",
"log_max_rotate": 99,
"repos": [{
"repo": "https://hg.mozilla.org/users/hwine_mozilla.com/repo-sync-tools",
"vcs": "hg",
}],
"job_name": "project-branches",
"conversion_dir": "project-branches",
"mapfile_name": "project-branches-mapfile",
"env": {
"PATH": "%(PATH)s:/usr/libexec/git-core",
},
"conversion_type": "project-branches",
"project_branches": PROJECT_BRANCHES,
"project_branch_repo_url": "https://hg.mozilla.org/projects/%(project)s",
"conversion_repos": CONVERSION_REPOS,
"remote_targets": {
"github-project-branches": {
"repo": "git@github.com:mozilla/gecko-projects.git",
"ssh_key": "~/.ssh/releng-github-id_rsa",
"vcs": "git",
"force_push": True,
},
},
"exes": {
# bug 828140 - shut https warnings up.
# http://kiln.stackexchange.com/questions/2816/mercurial-certificate-warning-certificate-not-verified-web-cacerts
"hg": [os.path.join(os.getcwd(), "build", "venv", "bin", "hg"), "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
"tooltool.py": [
os.path.join(os.getcwd(), "build", "venv", "bin", "python"),
os.path.join(os.getcwd(), "mozharness", "external_tools", "tooltool.py"),
],
},
"virtualenv_modules": [
"bottle==0.11.6",
"dulwich==0.9.0",
"ordereddict==1.1",
"hg-git==0.4.0-moz2",
"mapper==0.1",
"mercurial==2.6.3",
"mozfile==0.9",
"mozinfo==0.5",
"mozprocess==0.11",
],
"find_links": [
"http://pypi.pvt.build.mozilla.org/pub",
"http://pypi.pub.build.mozilla.org/pub",
],
"pip_index": False,
"combined_mapfile": "combined_gecko_mapfile",
"default_notify_from": "developer-services+%s@mozilla.org" % hostname,
"notify_config": [{
"to": "releng-ops-trial@mozilla.com",
"failure_only": False,
"skip_empty_messages": True,
}],
# Disallow sharing, since we want pristine .hg and .git directories.
"vcs_share_base": None,
"hg_share_base": None,
}

652
testing/mozharness/scripts/b2g_bumper.py
View File

@ -1,652 +0,0 @@
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
""" b2g_bumper.py
Updates a gecko repo with up to date information from B2G repositories.
In particular, it updates gaia.json which is used by B2G desktop builds,
and updates the XML manifests used by device builds.
This is to tie the external repository revisions to a visible gecko commit
which appears on TBPL, so sheriffs can blame the appropriate changes.
"""
import os
import sys
from multiprocessing.pool import ThreadPool
import subprocess
import time
from urlparse import urlparse
try:
import simplejson as json
assert json
except ImportError:
import json
sys.path.insert(1, os.path.dirname(sys.path[0]))
from mozharness.base.errors import HgErrorList
from mozharness.base.vcs.vcsbase import VCSScript
from mozharness.mozilla import repo_manifest
from mozharness.base.log import ERROR
from mozharness.mozilla.mapper import MapperMixin
class B2GBumper(VCSScript, MapperMixin):
config_options = [
[['--no-write'], {
'dest': 'do_write',
'action': 'store_const',
'const': False,
'help': 'disable writing in-tree manifests',
}],
[['--device'], {
'dest': 'device_override',
'help': 'specific device to process',
}],
]
def __init__(self, require_config_file=True):
super(B2GBumper, self).__init__(
config_options=self.config_options,
all_actions=[
'delete-git-ref-cache',
'import-git-ref-cache',
'clobber',
'check-treestatus',
'checkout-gecko',
'bump-gaia',
'checkout-manifests',
'massage-manifests',
'commit-manifests',
'push',
'push-loop',
'export-git-ref-cache',
],
default_actions=[
'push-loop',
],
require_config_file=require_config_file,
# Default config options
config={
'treestatus_base_url': 'https://treestatus.mozilla.org',
'log_max_rotate': 99,
'do_write': True,
}
)
# Mapping of device name to manifest
self.device_manifests = {}
# Cache of "%s:%s" % (remote url, refname) to revision hashes
self._git_ref_cache = {}
# File location for persisting _git_ref_cache dictionary above as a json file
self.git_ref_cache_file = self.config.get('git_ref_cache', os.path.join(self.query_abs_dirs()['abs_work_dir'], 'git_ref_cache.json'))
# Cache of new remotes to original upstreams
self._remote_mappings = {}
# What's the latest gaia revsion we have for hg
self.gaia_hg_revision = None
self.gaia_git_rev = None
# Helper methods {{{1
def query_abs_dirs(self):
if self.abs_dirs:
return self.abs_dirs
abs_dirs = super(B2GBumper, self).query_abs_dirs()
abs_dirs.update({
'manifests_dir':
os.path.join(abs_dirs['abs_work_dir'], 'manifests'),
'gecko_local_dir':
os.path.join(abs_dirs['abs_work_dir'],
self.config['gecko_local_dir']),
})
self.abs_dirs = abs_dirs
return self.abs_dirs
def query_manifest(self, device_name):
if device_name in self.device_manifests:
return self.device_manifests[device_name]
dirs = self.query_abs_dirs()
c = self.config
manifest_file = c['devices'][device_name].get('manifest_file',
'%s.xml' % device_name)
manifest_file = os.path.join(dirs['manifests_dir'], manifest_file)
self.info("Loading %s" % manifest_file)
manifest = repo_manifest.load_manifest(manifest_file)
self.device_manifests[device_name] = manifest
return manifest
def filter_projects(self, device_config, manifest):
for p in device_config['ignore_projects']:
removed = repo_manifest.remove_project(manifest, path=p)
if removed:
self.info("Removed %s" % removed.toxml())
def filter_groups(self, device_config, manifest):
for g in device_config.get('ignore_groups', []):
removed = repo_manifest.remove_group(manifest, g)
for r in removed:
self.info("Removed %s" % r.toxml())
def map_remotes(self, manifest):
def mapping_func(r):
orig_url = r.getAttribute('fetch')
m = repo_manifest.map_remote(r, self.config['repo_remote_mappings'])
self._remote_mappings[m.getAttribute('fetch')] = orig_url
return m
repo_manifest.rewrite_remotes(manifest, mapping_func)
def resolve_git_ref(self, remote_url, revision):
cache_key = "%s:%s" % (remote_url, revision)
cmd = ['git', 'ls-remote', remote_url, revision]
self.debug("Running %s" % cmd)
# Retry this a few times, in case there are network errors or somesuch
max_retries = 5
for _ in range(max_retries):
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if proc.wait() != 0:
self.warning("Returned %i - sleeping and retrying" %
proc.returncode)
self.warning("%s - got output: %s" % (cache_key, proc.stdout.read()))
time.sleep(30)
continue
output = proc.stdout.read()
self.info("%s - got output: %s" % (cache_key, output))
try:
abs_revision = output.split()[0].strip()
self._git_ref_cache[cache_key] = abs_revision
return abs_revision
except IndexError:
# Couldn't split the output properly
self.warning("no output from: git ls-remote %s %s" % (remote_url, revision))
return None
return None
def resolve_refs(self, manifest):
worker_pool = ThreadPool(20)
lookup_threads_by_project = {}
lookup_threads_by_parameters = {}
# Resolve refnames
for p in manifest.getElementsByTagName('project'):
name = p.getAttribute('name')
remote_url = repo_manifest.get_project_remote_url(manifest, p)
revision = repo_manifest.get_project_revision(manifest, p)
# commit ids are already done
if repo_manifest.is_commitid(revision):
self.debug("%s is already locked to %s; skipping" %
(name, revision))
continue
# gaia is special - make sure we're using the same revision we used
# for gaia.json
if self.gaia_hg_revision and p.getAttribute('path') == 'gaia' and revision == self.config['gaia_git_branch']:
git_rev = self.query_gaia_git_rev()
self.info("Using %s for gaia to match %s in gaia.json" % (git_rev, self.gaia_hg_revision))
p.setAttribute('revision', git_rev)
continue
# If there's no '/' in the revision, assume it's a head
if '/' not in revision:
revision = 'refs/heads/%s' % revision
cache_key = "%s:%s" % (remote_url, revision)
# Check to see if we've looked up this revision on this remote
# before. If we have, reuse the previous value rather than looking
# it up again. This will make sure revisions for the same ref name
# are consistent between devices, as long as they use the same
# remote/refname.
if cache_key in self._git_ref_cache:
abs_revision = self._git_ref_cache[cache_key]
self.debug(
"Reusing previous lookup %s -> %s" %
(cache_key, abs_revision))
p.setAttribute('revision', abs_revision)
continue
# Maybe a thread already exists for this lookup, even if the result has not
# yet been retrieved and placed in _git_ref_cache...
# Please note result.get() can be called multiple times without problems;
# the git command will only be executed once. Therefore we can associate many
# projects to the same thread result, without problems later when we call
# get() multiple times against the same thread result.
if cache_key in lookup_threads_by_parameters:
self.debug("Reusing currently running thread to look up %s" % cache_key)
lookup_threads_by_project[p] = lookup_threads_by_parameters.get(cache_key)
else:
async_result = worker_pool.apply_async(self.resolve_git_ref,
(remote_url, revision))
lookup_threads_by_parameters[cache_key] = async_result
lookup_threads_by_project[p] = async_result
# TODO: alert/notify on missing repositories
abort = False
failed = []
for p, result in lookup_threads_by_project.iteritems():
abs_revision = result.get(timeout=300)
remote_url = repo_manifest.get_project_remote_url(manifest, p)
revision = repo_manifest.get_project_revision(manifest, p)
if not abs_revision:
abort = True
self.error("Couldn't resolve reference %s %s" % (remote_url, revision))
failed.append(p)
p.setAttribute('revision', abs_revision)
if abort:
# Write message about how to set up syncing
default = repo_manifest.get_default(manifest)
for p in failed:
if p.hasAttribute('remote'):
remote = repo_manifest.get_remote(manifest, p.getAttribute('remote'))
else:
remote = repo_manifest.get_remote(manifest, default.getAttribute('remote'))
new_fetch_url = remote.getAttribute('fetch')
orig_fetch_url = self._remote_mappings[new_fetch_url]
name = p.getAttribute('name')
self.info("needs sync? %s/%s -> %s/%s" % (orig_fetch_url, name, new_fetch_url, name))
self.fatal("couldn't resolve some refs; exiting")
def query_manifest_path(self, device):
dirs = self.query_abs_dirs()
device_config = self.config['devices'][device]
manifest_file = os.path.join(
dirs['gecko_local_dir'],
'b2g', 'config',
device_config.get('gecko_device_dir', device),
'sources.xml')
return manifest_file
def hg_add(self, repo_path, path):
"""
Runs 'hg add' on path
"""
hg = self.query_exe('hg', return_type='list')
cmd = hg + ['add', path]
self.run_command(cmd, cwd=repo_path)
def hg_commit(self, repo_path, message):
"""
Commits changes in repo_path, with specified user and commit message
"""
user = self.config['hg_user']
hg = self.query_exe('hg', return_type='list')
cmd = hg + ['commit', '-u', user, '-m', message]
env = self.query_env(partial_env={'LANG': 'en_US.UTF-8'})
status = self.run_command(cmd, cwd=repo_path, env=env)
return status == 0
def hg_push(self, repo_path):
hg = self.query_exe('hg', return_type='list')
command = hg + ["push", "-e",
"ssh -oIdentityFile=%s -l %s" % (
self.config["ssh_key"], self.config["ssh_user"],
),
self.config["gecko_push_url"]]
status = self.run_command(command, cwd=repo_path,
error_list=HgErrorList)
if status != 0:
# We failed; get back to a known state so we can either retry
# or fail out and continue later.
self.run_command(hg + ["--config", "extensions.mq=",
"strip", "--no-backup", "outgoing()"],
cwd=repo_path)
self.run_command(hg + ["up", "-C"],
cwd=repo_path)
self.run_command(hg + ["--config", "extensions.purge=",
"purge", "--all"],
cwd=repo_path)
return False
return True
def _read_json(self, path):
if not os.path.exists(path):
self.error("%s doesn't exist!" % path)
return
contents = self.read_from_file(path)
try:
json_contents = json.loads(contents)
return json_contents
except ValueError:
self.error("%s is invalid json!" % path)
def get_revision_list(self, repo_config, prev_revision=None):
revision_list = []
url = repo_config['polling_url']
branch = repo_config.get('branch', 'default')
max_revisions = self.config['gaia_max_revisions']
dirs = self.query_abs_dirs()
if prev_revision:
# hgweb json-pushes hardcode
url += '&fromchange=%s' % prev_revision
file_name = os.path.join(dirs['abs_work_dir'],
'%s.json' % repo_config['repo_name'])
# might be nice to have a load-from-url option; til then,
# download then read
if self.retry(
self.download_file,
args=(url, ),
kwargs={'file_name': file_name},
error_level=ERROR,
sleeptime=0,
) != file_name:
return None
contents = self.read_from_file(file_name)
revision_dict = json.loads(contents)
if not revision_dict:
return []
# Discard any revisions not on the branch we care about.
for k in sorted(revision_dict, key=int): # numeric sort
v = revision_dict[k]
if v['changesets'][-1]['branch'] == branch:
revision_list.append(v)
# Limit the list to max_revisions.
return revision_list[:max_revisions]
def update_gaia_json(self, path,
hg_revision, hg_repo_path,
git_revision, git_repo,
):
""" Update path with repo_path + revision.
If the revision hasn't changed, don't do anything.
If the repo_path changes or the current json is invalid, error but don't fail.
"""
if not os.path.exists(path):
self.add_summary(
"%s doesn't exist; can't update with repo_path %s revision %s!" %
(path, hg_repo_path, hg_revision),
level=ERROR,
)
return -1
contents = self._read_json(path)
if contents:
if contents.get("repo_path") != hg_repo_path:
self.error("Current repo_path %s differs from %s!" % (str(contents.get("repo_path")), hg_repo_path))
if contents.get("revision") == hg_revision:
self.info("Revision %s is the same. No action needed." % hg_revision)
self.add_summary("Revision %s is unchanged for repo_path %s." % (hg_revision, hg_repo_path))
return 0
contents = {
"repo_path": hg_repo_path,
"revision": hg_revision,
"git": {
"remote": git_repo,
"branch": "",
"git_revision": git_revision,
}
}
if self.write_to_file(path, json.dumps(contents, indent=4) + "\n") != path:
self.add_summary(
"Unable to update %s with new revision %s!" % (path, hg_revision),
level=ERROR,
)
return -2
def build_commit_message(self, revision_list, repo_name, repo_url):
revisions = []
comments = ''
for revision_config in reversed(revision_list):
for changeset_config in reversed(revision_config['changesets']):
revisions.append(changeset_config['node'])
comments += "\n========\n"
comments += u'\n%s/rev/%s\nAuthor: %s\nDesc: %s\n' % (
repo_url,
changeset_config['node'][:12],
changeset_config['author'],
changeset_config['desc'],
)
message = 'Bumping gaia.json for %d %s revision(s) a=gaia-bump\n' % (
len(revisions),
repo_name
)
message += comments
message = message.encode("utf-8")
return message
def query_treestatus(self):
"Return True if we can land based on treestatus"
c = self.config
dirs = self.query_abs_dirs()
tree = c.get('treestatus_tree', os.path.basename(c['gecko_pull_url'].rstrip("/")))
treestatus_url = "%s/%s?format=json" % (c['treestatus_base_url'], tree)
treestatus_json = os.path.join(dirs['abs_work_dir'], 'treestatus.json')
if not os.path.exists(dirs['abs_work_dir']):
self.mkdir_p(dirs['abs_work_dir'])
if self.download_file(treestatus_url, file_name=treestatus_json) != treestatus_json:
# Failed to check tree status...assume we can land
self.info("failed to check tree status - assuming we can land")
return True
treestatus = self._read_json(treestatus_json)
if treestatus['status'] != 'closed':
self.info("treestatus is %s - assuming we can land" % repr(treestatus['status']))
return True
return False
def query_devices(self):
c = self.config
override = c.get('device_override')
if override:
return {override: c['devices'][override]}
else:
return c['devices']
def query_gaia_git_rev(self):
"""Returns (and caches) the git revision for gaia corresponding to the
latest hg revision on our branch."""
if not self.gaia_hg_revision:
return None
if not self.gaia_git_rev:
self.gaia_git_rev = self.query_mapper_git_revision(
self.config['mapper_url'],
self.config['gaia_mapper_project'],
self.gaia_hg_revision,
)
return self.gaia_git_rev
# Actions {{{1
def check_treestatus(self):
if not self.query_treestatus():
self.info("breaking early since treestatus is closed")
sys.exit(0)
def checkout_gecko(self):
c = self.config
dirs = self.query_abs_dirs()
dest = dirs['gecko_local_dir']
repos = [{
'repo': c['gecko_pull_url'],
'tag': c.get('gecko_tag', 'default'),
'dest': dest,
'vcs': 'hgtool',
'hgtool_base_bundle_urls': c.get('hgtool_base_bundle_urls'),
}]
self.vcs_checkout_repos(repos)
def checkout_manifests(self):
c = self.config
dirs = self.query_abs_dirs()
repos = [
{'vcs': 'gittool',
'repo': c['manifests_repo'],
'revision': c['manifests_revision'],
'dest': dirs['manifests_dir']},
]
self.vcs_checkout_repos(repos)
def massage_manifests(self):
"""
For each device in config['devices'], we'll strip projects mentioned in
'ignore_projects', or that have group attribute mentioned in
'filter_groups'.
We'll also map remote urls
Finally, we'll resolve absolute refs for projects that aren't fully
specified.
"""
for device, device_config in self.query_devices().items():
self.info("Massaging manifests for %s" % device)
manifest = self.query_manifest(device)
self.filter_projects(device_config, manifest)
self.filter_groups(device_config, manifest)
self.map_remotes(manifest)
self.resolve_refs(manifest)
repo_manifest.cleanup(manifest)
self.device_manifests[device] = manifest
manifest_path = self.query_manifest_path(device)
manifest_xml = manifest.toxml()
if not manifest_xml.endswith("\n"):
manifest_xml += "\n"
if self.config['do_write']:
self.mkdir_p(os.path.dirname(manifest_path))
self.write_to_file(manifest_path, manifest_xml)
def commit_manifests(self):
dirs = self.query_abs_dirs()
repo_path = dirs['gecko_local_dir']
for device, device_config in self.query_devices().items():
manifest_path = self.query_manifest_path(device)
self.hg_add(repo_path, manifest_path)
message = "Bumping manifests a=b2g-bump"
return self.hg_commit(repo_path, message)
def bump_gaia(self):
dirs = self.query_abs_dirs()
repo_path = dirs['gecko_local_dir']
gaia_json_path = os.path.join(repo_path,
self.config['gaia_revision_file'])
contents = self._read_json(gaia_json_path)
# Get the list of changes
if contents:
prev_revision = contents.get('revision')
self.gaia_hg_revision = prev_revision
else:
prev_revision = None
polling_url = "%s/json-pushes?full=1" % self.config['gaia_repo_url']
repo_config = {
'polling_url': polling_url,
'branch': self.config.get('gaia_branch', 'default'),
'repo_name': 'gaia',
}
revision_list = self.get_revision_list(repo_config=repo_config,
prev_revision=prev_revision)
if not revision_list:
# No changes
return False
# Update the gaia.json with the list of changes
hg_gaia_repo_path = urlparse(self.config['gaia_repo_url']).path.lstrip('/')
hg_revision = revision_list[-1]['changesets'][-1]['node']
self.gaia_hg_revision = hg_revision
git_revision = self.query_gaia_git_rev()
git_gaia_repo = self.config['gaia_git_repo']
self.update_gaia_json(gaia_json_path,
hg_revision, hg_gaia_repo_path,
git_revision, git_gaia_repo,
)
# Commit
message = self.build_commit_message(revision_list, 'gaia',
self.config['gaia_repo_url'])
self.hg_commit(repo_path, message)
return True
def push(self):
dirs = self.query_abs_dirs()
repo_path = dirs['gecko_local_dir']
return self.hg_push(repo_path)
def push_loop(self):
max_retries = 5
for _ in range(max_retries):
changed = False
if not self.query_treestatus():
# Tree is closed; exit early to avoid a bunch of wasted time
self.info("breaking early since treestatus is closed")
break
self.checkout_gecko()
if not self.config.get('skip_gaia_json') and self.bump_gaia():
changed = True
self.checkout_manifests()
self.massage_manifests()
if self.commit_manifests():
changed = True
if not changed:
# Nothing changed, we're all done
self.info("No changes - all done")
break
if self.push():
# We did it! Hurray!
self.info("Great success!")
break
# If we're here, then the push failed. It also stripped any
# outgoing commits, so we should be in a pristine state again
# Empty our local cache of manifests so they get loaded again next
# time through this loop. This makes sure we get fresh upstream
# manifests, and avoids problems like bug 979080
self.device_manifests = {}
# Sleep before trying again
self.info("Sleeping 60 before trying again")
time.sleep(60)
else:
self.fatal("Didn't complete successfully (hit max_retries)")
def import_git_ref_cache(self):
""" This action imports the git ref cache created during a previous run. This is
useful for sharing the cache across multiple branches (for example).
"""
if os.path.exists(self.git_ref_cache_file):
self._git_ref_cache = self._read_json(self.git_ref_cache_file)
def export_git_ref_cache(self):
""" This action exports the git ref cache created during this run. This is useful
for sharing the cache across multiple branches (for example).
"""
if self.write_to_file(self.git_ref_cache_file, json.dumps(self._git_ref_cache, sort_keys=True, indent=4) + "\n") != self.git_ref_cache_file:
self.add_summary(
"Unable to update %s with git ref cache" % self.git_ref_cache_file,
level=ERROR,
)
return -2
def delete_git_ref_cache(self):
""" Used to delete the git ref cache from the file system. The cache can be used
to persist git ls-remote lookup results, for example to reuse them between b2g bumper
runs. Since the results are stale and do not get updated, the cache should be
periodically deleted, so that the new refs can be fetched. The cache can also be used
across branches/devices.
"""
self.log("Deleting git ls-remote look-up cache file ('%s')...")
os.remove(self.git_ref_cache_file)
# __main__ {{{1
if __name__ == '__main__':
bumper = B2GBumper()
bumper.run_and_exit()

744
testing/mozharness/scripts/vcs-sync/initial_beagle.py
View File

@ -1,744 +0,0 @@
#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""initial_beagle.py
Multi-repo m-c hg->git initial conversion with cvs prepending, specifically for
gecko.git and beagle support.
Separated from hg_git.py for a) simplifying hg_git.py for its main purpose,
and b) somewhat protecting the initial conversion steps from future edits.
"""
from copy import deepcopy
import mmap
import os
import re
import sys
sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
import mozharness
external_tools_path = os.path.join(
os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
'external_tools',
)
from mozharness.base.errors import HgErrorList, GitErrorList, TarErrorList
from mozharness.base.log import INFO, FATAL
from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
from mozharness.base.transfer import TransferMixin
from mozharness.base.vcs.vcsbase import VCSScript
from mozharness.mozilla.tooltool import TooltoolMixin
# HgGitScript {{{1
class HgGitScript(VirtualenvMixin, TooltoolMixin, TransferMixin, VCSScript):
""" Beagle-oriented hg->git script (lots of mozilla-central hardcodes;
assumption that we're going to be importing lots of branches).
Beagle is a git repo of mozilla-central, with full cvs history,
and a number of developer-oriented repositories and branches added.
The partner-oriented gecko.git could also be incorporated into this
script with some changes.
"""
mapfile_binary_search = None
successful_repos = [] # Unused; for notify() capability with vcs_sync.py
def __init__(self, require_config_file=True):
super(HgGitScript, self).__init__(
config_options=virtualenv_config_options,
all_actions=[
'clobber',
'create-virtualenv',
'pull',
'create-stage-mirror',
'create-work-mirror',
'initial-conversion',
'prepend-cvs',
'fix-tags',
'notify',
],
# These default actions are the update loop that we run after the
# initial steps to create the work mirror with all the branches +
# cvs history have been run.
require_config_file=require_config_file
)
# Helper methods {{{1
def query_abs_dirs(self):
""" Define paths.
"""
if self.abs_dirs:
return self.abs_dirs
abs_dirs = super(HgGitScript, self).query_abs_dirs()
abs_dirs['abs_cvs_history_dir'] = os.path.join(
abs_dirs['abs_work_dir'], 'mozilla-cvs-history')
abs_dirs['abs_conversion_dir'] = os.path.join(
abs_dirs['abs_work_dir'], 'conversion',
self.config['conversion_dir']
)
abs_dirs['abs_source_dir'] = os.path.join(
abs_dirs['abs_work_dir'], 'stage_source')
abs_dirs['abs_repo_sync_tools_dir'] = os.path.join(
abs_dirs['abs_work_dir'], 'repo-sync-tools')
abs_dirs['abs_git_rewrite_dir'] = os.path.join(
abs_dirs['abs_work_dir'], 'mc-git-rewrite')
abs_dirs['abs_target_dir'] = os.path.join(abs_dirs['abs_work_dir'],
'target')
self.abs_dirs = abs_dirs
return self.abs_dirs
def query_all_repos(self):
""" Very simple method, but we need this concatenated list many times
throughout the script.
"""
return [self.config['initial_repo']]
def _update_stage_repo(self, repo_config, retry=True, clobber=False):
""" Update a stage repo.
The stage mirror is a buffer clean clone of repositories.
The logic behind this is that we get occasional corruption from
|hg pull|. It's much less time-consuming to detect this in
a clean clone, and reclone, than to detect this in a working
conversion directory, and try to repair or reclone+reconvert.
We pull the stage mirror into the work mirror, where the
conversion
is done.
"""
hg = self.query_exe('hg', return_type='list')
dirs = self.query_abs_dirs()
source_dest = os.path.join(dirs['abs_source_dir'],
repo_config['repo_name'])
if clobber:
self.rmtree(source_dest)
if not os.path.exists(source_dest):
if self.retry(
self.run_command,
args=(hg + ['clone', '--noupdate', repo_config['repo'],
source_dest], ),
kwargs={
'output_timeout': 15 * 60,
'cwd': dirs['abs_work_dir'],
'error_list': HgErrorList,
},
):
if retry:
return self._update_stage_repo(
repo_config, retry=False, clobber=True)
else:
self.fatal("Can't clone %s!" % repo_config['repo'])
cmd = hg + ['pull']
if self.retry(
self.run_command,
args=(cmd, ),
kwargs={
'output_timeout': 15 * 60,
'cwd': source_dest,
},
):
if retry:
return self._update_stage_repo(
repo_config, retry=False, clobber=True)
else:
self.fatal("Can't pull %s!" % repo_config['repo'])
# commenting out hg verify since it takes ~5min per repo; hopefully
# exit codes will save us
# if self.run_command(hg + ["verify"], cwd=source_dest):
# if retry:
# return self._update_stage_repo(repo_config, retry=False, clobber=True)
# else:
# self.fatal("Can't verify %s!" % source_dest)
def _check_initial_git_revisions(self, repo_path, expected_sha1,
expected_sha2):
""" Verify that specific git revisions match expected shas.
This involves some hardcodes, which is unfortunate, but they save
time, especially since we're hardcoding mozilla-central behavior
anyway.
"""
git = self.query_exe('git', return_type='list')
output = self.get_output_from_command(
git + ['log', '--oneline', '--grep', '374866'],
cwd=repo_path
)
# hardcode test
if not output:
self.fatal("No output from git log!")
rev = output.split(' ')[0]
if not rev.startswith(expected_sha1):
self.fatal("Output doesn't match expected sha %s for initial hg commit: %s" % (expected_sha1, str(output)))
output = self.get_output_from_command(
git + ['log', '-n', '1', '%s^' % rev],
cwd=repo_path
)
if not output:
self.fatal("No output from git log!")
rev = output.splitlines()[0].split(' ')[1]
if rev != expected_sha2:
self.fatal("Output rev %s doesn't show expected rev %s:\n\n%s" % (rev, expected_sha2, output))
def munge_mapfile(self):
""" From https://github.com/ehsan/mozilla-history-tools/blob/master/initial_conversion/translate_git-mapfile.py
"""
self.info("Updating pre-cvs mapfile...")
dirs = self.query_abs_dirs()
orig_mapfile = os.path.join(dirs['abs_upload_dir'], 'pre-cvs-mapfile')
conversion_dir = dirs['abs_conversion_dir']
mapfile = os.path.join(dirs['abs_work_dir'], 'post-cvs-mapfile')
mapdir = os.path.join(dirs['abs_git_rewrite_dir'], 'map')
orig_mapfile_fh = open(orig_mapfile, "r")
mapfile_fh = open(mapfile, "w")
for line in orig_mapfile_fh:
tokens = line.split(" ")
if len(tokens) == 2:
git_sha = tokens[0].strip()
hg_sha = tokens[1].strip()
new_path = os.path.join(mapdir, git_sha)
if os.path.exists(new_path):
translated_git_sha = open(new_path).read().strip()
print >>mapfile_fh, "%s %s" % (translated_git_sha, hg_sha)
else:
print >>mapfile_fh, "%s %s" % (git_sha, hg_sha)
orig_mapfile_fh.close()
mapfile_fh.close()
self.copyfile(
mapfile,
os.path.join(conversion_dir, '.hg', 'git-mapfile'),
error_level=FATAL,
)
self.copy_to_upload_dir(mapfile, dest="post-cvs-mapfile",
log_level=INFO)
def make_repo_bare(self, path, tmpdir=None):
""" Since we do a |git checkout| in prepend_cvs(), and later want
a bare repo.
"""
self.info("Making %s/.git a bare repo..." % path)
for p in (path, os.path.join(path, ".git")):
if not os.path.exists(p):
self.error("%s doesn't exist! Skipping..." % p)
if tmpdir is None:
tmpdir = os.path.dirname(os.path.abspath(path))
git = self.query_exe("git", return_type="list")
for dirname in (".git", ".hg"):
if os.path.exists(os.path.join(path, dirname)):
self.move(
os.path.join(path, dirname),
os.path.join(tmpdir, dirname),
error_level=FATAL,
)
self.rmtree(path, error_level=FATAL)
self.mkdir_p(path)
for dirname in (".git", ".hg"):
if os.path.exists(os.path.join(tmpdir, dirname)):
self.move(
os.path.join(tmpdir, dirname),
os.path.join(path, dirname),
error_level=FATAL,
)
self.run_command(
git + ['--git-dir', os.path.join(path, ".git"),
'config', '--bool', 'core.bare', 'true'],
halt_on_failure=True,
)
def _fix_tags(self, conversion_dir, git_rewrite_dir):
""" Ehsan's git tag fixer, ported from bash.
`` Git's history rewriting is not smart about preserving the tags in
your repository, so you would end up with tags which point to
commits in the old history line. If you push your repository to
some other repository for example, all of the tags in the target
repository would be invalid, since they would be pointing to
commits that don't exist in that repository. ''
https://github.com/ehsan/mozilla-history-tools/blob/master/initial_conversion/translate_git_tags.sh
"""
self.info("Fixing tags...")
git = self.query_exe('git', return_type='list')
output = self.get_output_from_command(
git + ['for-each-ref'],
cwd=conversion_dir,
halt_on_failure=True,
)
for line in output.splitlines():
old_sha1, the_rest = line.split(' ')
git_type, name = the_rest.split(' ')
if git_type == 'commit' and name.startswith('refs/tags'):
path = os.path.join(git_rewrite_dir, 'map', old_sha1)
if os.path.exists(path):
new_sha1 = self.read_from_file(path).rstrip()
self.run_command(
git + ['update-ref', name,
new_sha1, old_sha1],
cwd=conversion_dir,
error_list=GitErrorList,
halt_on_failure=True,
)
def _do_push_repo(self, base_command, refs_list=None, kwargs=None):
""" Helper method for _push_repo() since it has to be able to break
out of the target_repo list loop, and the commands loop borks that.
"""
commands = []
if refs_list:
while len(refs_list) > 10:
commands.append(base_command + refs_list[0:10])
refs_list = refs_list[10:]
commands.append(base_command + refs_list)
else:
commands = [base_command]
if kwargs is None:
kwargs = {}
for command in commands:
# Do the push, with retry!
if self.retry(
self.run_command,
args=(command, ),
kwargs=kwargs,
):
return -1
def _push_repo(self, repo_config):
""" Push a repo to a path ("test_push") or remote server.
This was meant to be a cross-vcs method, but currently only
covers git pushes.
"""
dirs = self.query_abs_dirs()
conversion_dir = dirs['abs_conversion_dir']
source_dir = os.path.join(dirs['abs_source_dir'], repo_config['repo_name'])
git = self.query_exe('git', return_type='list')
hg = self.query_exe('hg', return_type='list')
return_status = ''
for target_config in repo_config['targets']:
if target_config.get("vcs", "git") == "git":
base_command = git + ['push']
env = {}
if target_config.get("force_push"):
base_command.append("-f")
if target_config.get("test_push"):
target_name = os.path.join(
dirs['abs_target_dir'], target_config['target_dest'])
base_command.append(target_name)
else:
target_name = target_config['target_dest']
remote_config = self.config.get('remote_targets', {}).get(target_name)
if not remote_config:
self.fatal("Can't find %s in remote_targets!" % target_name)
base_command.append(remote_config['repo'])
# Allow for using a custom git ssh key.
env['GIT_SSH_KEY'] = remote_config['ssh_key']
env['GIT_SSH'] = os.path.join(external_tools_path, 'git-ssh-wrapper.sh')
# Allow for pushing a subset of repo branches to the target.
# If we specify that subset, we can also specify different
# names for those branches (e.g. b2g18 -> master for a
# standalone b2g18 repo)
# We query hg for these because the conversion dir will have
# branches from multiple hg repos, and the regexes may match
# too many things.
refs_list = []
branch_map = self.query_branches(
target_config.get('branch_config', repo_config.get('branch_config', {})),
source_dir,
)
# If the target_config has a branch_config, the key is the
# local git branch and the value is the target git branch.
if target_config.get("branch_config"):
for (branch, target_branch) in branch_map.items():
refs_list += ['+refs/heads/%s:refs/heads/%s' % (branch, target_branch)]
# Otherwise the key is the hg branch and the value is the git
# branch; use the git branch for both local and target git
# branch names.
else:
for (hg_branch, git_branch) in branch_map.items():
refs_list += ['+refs/heads/%s:refs/heads/%s' % (git_branch, git_branch)]
# Allow for pushing a subset of tags to the target, via name or
# regex. Again, query hg for this list because the conversion
# dir will contain tags from multiple hg repos, and the regexes
# may match too many things.
tag_config = target_config.get('tag_config', repo_config.get('tag_config', {}))
if tag_config.get('tags'):
for (tag, target_tag) in tag_config['tags'].items():
refs_list += ['+refs/tags/%s:refs/tags/%s' % (tag, target_tag)]
if tag_config.get('tag_regexes'):
regex_list = []
for regex in tag_config['tag_regexes']:
regex_list.append(re.compile(regex))
tag_list = self.get_output_from_command(
hg + ['tags'],
cwd=source_dir,
)
for tag_line in tag_list.splitlines():
if not tag_line:
continue
tag_parts = tag_line.split()
if not tag_parts:
self.warning("Bogus tag_line? %s" % str(tag_line))
continue
tag_name = tag_parts[0]
for regex in regex_list:
if regex.search(tag_name) is not None:
refs_list += ['+refs/tags/%s:refs/tags/%s' % (tag_name, tag_name)]
continue
error_msg = "%s: Can't push %s to %s!\n" % (repo_config['repo_name'], conversion_dir, target_name)
if self._do_push_repo(
base_command,
refs_list=refs_list,
kwargs={
'output_timeout': target_config.get("output_timeout", 30 * 60),
'cwd': os.path.join(conversion_dir, '.git'),
'error_list': GitErrorList,
'partial_env': env,
}
):
if target_config.get("test_push"):
error_msg += "This was a test push that failed; not proceeding any further with %s!\n" % repo_config['repo_name']
self.error(error_msg)
return_status += error_msg
if target_config.get("test_push"):
break
else:
# TODO write hg
error_msg = "%s: Don't know how to deal with vcs %s!\n" % (
target_config['target_dest'], target_config['vcs'])
self.error(error_msg)
return_status += error_msg
return return_status
def _query_mapped_revision(self, revision=None, mapfile=None):
""" Use the virtualenv mapper module to search a mapfile for a
revision.
"""
if not callable(self.mapfile_binary_search):
site_packages_path = self.query_python_site_packages_path()
sys.path.append(os.path.join(site_packages_path, 'mapper'))
try:
from bsearch import mapfile_binary_search
global log
log = self.log_obj
self.mapfile_binary_search = mapfile_binary_search
except ImportError, e:
self.fatal("Can't import mapfile_binary_search! %s\nDid you create-virtualenv?" % str(e))
# I wish mapper did this for me, but ...
fd = open(mapfile, 'rb')
m = mmap.mmap(fd.fileno(), 0, mmap.MAP_PRIVATE, mmap.PROT_READ)
return self.mapfile_binary_search(m, revision)
def _post_fatal(self, message=None, exit_code=None):
""" After we call fatal(), run this method before exiting.
"""
if 'notify' in self.actions:
self.notify(message=message, fatal=True)
self.copy_logs_to_upload_dir()
def query_branches(self, branch_config, repo_path, vcs='hg'):
""" Given a branch_config of branches and branch_regexes, return
a dict of existing branch names to target branch names.
"""
branch_map = {}
if "branches" in branch_config:
branch_map = deepcopy(branch_config['branches'])
if "branch_regexes" in branch_config:
regex_list = list(branch_config['branch_regexes'])
full_branch_list = []
if vcs == 'hg':
hg = self.query_exe("hg", return_type="list")
# This assumes we always want closed branches as well.
# If not we may need more options.
output = self.get_output_from_command(
hg + ['branches', '-a'],
cwd=repo_path
)
if output:
for line in output.splitlines():
full_branch_list.append(line.split()[0])
elif vcs == 'git':
git = self.query_exe("git", return_type="list")
output = self.get_output_from_command(
git + ['branch', '-l'],
cwd=repo_path
)
if output:
for line in output.splitlines():
full_branch_list.append(line.replace('*', '').split()[0])
for regex in regex_list:
for branch in full_branch_list:
m = re.search(regex, branch)
if m:
# Don't overwrite branch_map[branch] if it exists
branch_map.setdefault(branch, branch)
return branch_map
# Actions {{{1
def create_stage_mirror(self):
""" Rather than duplicate the logic here and in update_stage_mirror(),
just call it.
We could just create the initial_repo stage mirror here, but
there's no real harm in cloning all repos here either. Putting
the time hit in the one-time-setup, rather than the first update
loop, makes sense.
"""
for repo_config in self.query_all_repos():
self._update_stage_repo(repo_config)
def write_hggit_hgrc(self, dest):
# Update .hg/hgrc, if not already updated
hgrc = os.path.join(dest, '.hg', 'hgrc')
contents = ''
if os.path.exists(hgrc):
contents = self.read_from_file(hgrc)
if 'hggit=' not in contents:
hgrc_update = """[extensions]
hggit=
[git]
intree=1
"""
self.write_to_file(hgrc, hgrc_update, open_mode='a')
def create_work_mirror(self):
""" Create the work_mirror, initial_repo only, from the stage_mirror.
This is where the conversion will occur.
"""
hg = self.query_exe("hg", return_type="list")
git = self.query_exe("git", return_type="list")
dirs = self.query_abs_dirs()
repo_config = deepcopy(self.config['initial_repo'])
work_dest = dirs['abs_conversion_dir']
source_dest = os.path.join(
dirs['abs_source_dir'], repo_config['repo_name'])
if not os.path.exists(work_dest):
self.run_command(hg + ["init", work_dest],
halt_on_failure=True)
self.run_command(hg + ["pull", source_dest],
cwd=work_dest,
error_list=HgErrorList,
halt_on_failure=True)
# The revision 82e4f1b7bbb6e30a635b49bf2107b41a8c26e3d2
# reacts poorly to git-filter-branch-keep-rewrites (in
# prepend-cvs), resulting in diverging shas.
# To avoid this, strip back to 317fe0f314ab so the initial conversion
# doesn't include 82e4f1b7bbb6e30a635b49bf2107b41a8c26e3d2, and
# git-filter-branch-keep-rewrites is never run against this
# revision. This takes 3 strips, due to forking/merging.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=847727#c40 through
# https://bugzilla.mozilla.org/show_bug.cgi?id=847727#c60
# Also, yay hardcodes!
for hg_revision in ("26cb30a532a1", "aad29aa89237", "9f2fa4839e98", "f8d0784186b7"):
self.run_command(hg + ["--config", "extensions.mq=", "strip",
"--no-backup", hg_revision],
cwd=work_dest,
error_list=HgErrorList,
halt_on_failure=True)
# Make sure 317fe0f314ab is the only head!
self.info("Making sure we've stripped m-c down to a single head 317fe0f314ab...")
output = self.get_output_from_command(hg + ["heads"],
cwd=work_dest,
halt_on_failure=True)
for line in output.splitlines():
if line.startswith("changeset:") and not line.endswith("317fe0f314ab"):
self.fatal("Found a head that is not 317fe0f314ab! hg strip or <h<surkov will show up again!\n\n%s" % output)
# Create .git for conversion, if it doesn't exist
git_dir = os.path.join(work_dest, '.git')
if not os.path.exists(git_dir):
self.run_command(git + ['init'], cwd=work_dest)
self.run_command(
git + ['--git-dir', git_dir, 'config', 'gc.auto', '0'],
cwd=work_dest
)
self.write_hggit_hgrc(work_dest)
def initial_conversion(self):
""" Run the initial hg-git conversion of the work_mirror.
This will create a git m-c repository without cvs history.
"""
hg = self.query_exe("hg", return_type="list")
dirs = self.query_abs_dirs()
repo_config = deepcopy(self.config['initial_repo'])
source = os.path.join(dirs['abs_source_dir'], repo_config['repo_name'])
dest = dirs['abs_conversion_dir']
# bookmark all the branches in the repo_config, potentially
# renaming them.
# This follows a slightly different workflow than elsewhere; I don't
# want to fiddle with this logic more than I have to.
for (branch, target_branch) in repo_config.get('branch_config', {}).get('branches', {}).items():
output = self.get_output_from_command(
hg + ['id', '-r', branch], cwd=dest)
if output:
rev = output.split(' ')[0]
self.run_command(
hg + ['bookmark', '-f', '-r', rev, target_branch],
cwd=dest,
error_list=HgErrorList,
halt_on_failure=True,
)
# bookmark all the other branches, with the same name.
output = self.get_output_from_command(hg + ['branches', '-c'], cwd=source)
for line in output.splitlines():
branch_name = line.split(' ')[0]
if branch_name in repo_config.get('branch_config', {}).get('branches', {}):
continue
self.run_command(
hg + ['bookmark', '-f', '-r', branch_name, branch_name],
cwd=dest,
error_list=HgErrorList,
halt_on_failure=True,
)
# Do the conversion! This will take a day or so.
self.retry(
self.run_command,
args=(hg + ['-v', 'gexport'], ),
kwargs={
'output_timeout': 15 * 60,
'cwd': dest,
'error_list': HgErrorList,
},
error_level=FATAL,
)
# Save the pre-cvs mapfile.
self.copy_to_upload_dir(os.path.join(dest, '.hg', 'git-mapfile'),
dest="pre-cvs-mapfile", log_level=INFO)
def prepend_cvs(self):
""" Prepend converted CVS history to the converted git repository,
then munge the branches with git-filter-branch-keep-rewrites
to adjust the shas.
This step can take on the order of a week of compute time.
"""
dirs = self.query_abs_dirs()
git = self.query_exe('git', return_type='list')
conversion_dir = dirs['abs_conversion_dir']
git_conversion_dir = os.path.join(conversion_dir, '.git')
grafts_file = os.path.join(git_conversion_dir, 'info', 'grafts')
map_dir = os.path.join(git_conversion_dir, '.git-rewrite', 'map')
if not os.path.exists(dirs["abs_cvs_history_dir"]):
# gd2 doesn't have access to tooltool :(
#manifest_path = self.create_tooltool_manifest(self.config['cvs_manifest'])
#if self.tooltool_fetch(manifest_path, output_dir=dirs['abs_work_dir']):
# self.fatal("Unable to download cvs history via tooltool!")
# Temporary workaround
self.copyfile(
self.config['cvs_history_tarball'],
os.path.join(dirs['abs_work_dir'], "mozilla-cvs-history.tar.bz2")
)
self.run_command(
["tar", "xjvf", "mozilla-cvs-history.tar.bz2"],
cwd=dirs["abs_work_dir"],
error_list=TarErrorList,
halt_on_failure=True
)
# We need to git checkout, or git thinks we've removed all the files
# without committing
self.run_command(git + ["checkout"], cwd=conversion_dir)
self.run_command(
'cp ' + os.path.join(dirs['abs_cvs_history_dir'], 'objects',
'pack', '*') + ' .',
cwd=os.path.join(git_conversion_dir, 'objects', 'pack')
)
self._check_initial_git_revisions(dirs['abs_cvs_history_dir'], 'e230b03',
'3ec464b55782fb94dbbb9b5784aac141f3e3ac01')
self._check_initial_git_revisions(conversion_dir, '4b3fd9',
'2514a423aca5d1273a842918589e44038d046a51')
self.write_to_file(grafts_file,
'2514a423aca5d1273a842918589e44038d046a51 3ec464b55782fb94dbbb9b5784aac141f3e3ac01')
# This script is modified from git-filter-branch from git.
# https://people.mozilla.com/~hwine/tmp/vcs2vcs/notes.html#initial-conversion
# We may need to update this script if we update git.
# Currently, due to the way the script outputs (with \r but not \n),
# mozharness doesn't output anything for a number of ours. This
# prevents using the output_timeout run_command() option (or, if we
# did, it would have to be many hours long).
env = deepcopy(self.config.get('env', {}))
git_filter_branch = os.path.join(
dirs['abs_repo_sync_tools_dir'],
'git-filter-branch-keep-rewrites'
)
self.run_command(
[git_filter_branch, '--',
'3ec464b55782fb94dbbb9b5784aac141f3e3ac01..HEAD'],
partial_env=env,
cwd=conversion_dir,
halt_on_failure=True
)
# The self.move() will break if this dir already exists.
# This rmtree() could move up to a preflight_prepend_cvs() method, or
# near the beginning, if desired.
self.rmtree(dirs['abs_git_rewrite_dir'])
self.move(os.path.join(conversion_dir, '.git-rewrite'),
dirs['abs_git_rewrite_dir'],
error_level=FATAL)
self.make_repo_bare(conversion_dir)
branch_list = self.get_output_from_command(
git + ['branch'],
cwd=git_conversion_dir,
)
for branch in branch_list.splitlines():
if branch.startswith('*'):
# specifically deal with |* master|
continue
branch = branch.strip()
self.run_command(
[git_filter_branch, '-f', '--',
'3ec464b55782fb94dbbb9b5784aac141f3e3ac01..%s' % branch],
partial_env=env,
cwd=git_conversion_dir,
halt_on_failure=True
)
# tar backup after mid-prepend-cvs gd2 reboot in bug 894225.
if os.path.exists(map_dir):
if self.config.get("backup_dir"):
self.mkdir_p(self.config['backup_dir'])
self.run_command(
["tar", "cjvf",
os.path.join(self.config["backup_dir"],
"prepend-cvs-%s.tar.bz2" % branch),
map_dir],
)
self.run_command(
['rsync', '-azv', os.path.join(map_dir, '.'),
os.path.join(dirs['abs_git_rewrite_dir'], 'map', '.')],
halt_on_failure=True
)
self.rmtree(os.path.join(git_conversion_dir, '.git-rewrite'),
error_level=FATAL)
self.rmtree(grafts_file, error_level=FATAL)
self.munge_mapfile()
def fix_tags(self):
""" Fairly fast action that points each existing tag to the new
cvs-prepended sha.
Then we git gc to get rid of old shas. This doesn't specifically
belong in this action, though it's the right spot in the workflow.
We have to gc to get rid of the <h<surkov email issue that
git-filter-branch-keep-rewrites gets rid of in the new shas.
"""
dirs = self.query_abs_dirs()
git = self.query_exe("git", return_type="list")
conversion_dir = dirs['abs_conversion_dir']
self._fix_tags(
os.path.join(conversion_dir, '.git'),
dirs['abs_git_rewrite_dir']
)
self.run_command(
git + ['gc', '--aggressive'],
cwd=os.path.join(conversion_dir, '.git'),
error_list=GitErrorList,
halt_on_failure=True,
)
# __main__ {{{1
if __name__ == '__main__':
conversion = HgGitScript()
conversion.run()

1105
testing/mozharness/scripts/vcs-sync/vcs_sync.py
View File

File diff suppressed because it is too large Load Diff