CI: Modifications to enable reproducible builds

Added default source revision file name to common.py.
Modified the integration_into_manifest function to set the
corresponding subproject hash in the SRCREV.conf file if the
repository is not found in the default.xml file. The
merge_into_manifest function now works even if the names of
the master branches in the manifest and in the project
repository are different.

BCS 746-000016

common.py

@@ -11,6 +11,7 @@ from gitlab.v4.objects import MergeRequest
 
 
 manifest_file = "default.xml"
+srcrev_file = "SRCREV.conf"
 pending_states = ["created", "waiting_for_resource", "preparing", "pending", "running"]
 
 

integrate_into_manifest.py

@@ -4,6 +4,7 @@ import common
 import argparse
 import sys
 import tempfile
+import re
 from pathlib import Path
 from furl import furl
 from git import GitCommandError, Repo
@@ -15,6 +16,8 @@ def integrate_into_manifest(
     manifest_project,
     integration_base,
     manifest_file,
+    srcrev_file,
+    recipe_name,
     project,
     merge_request,
 ):
@@ -22,6 +25,7 @@ def integrate_into_manifest(
 
     with tempfile.TemporaryDirectory() as manifest_dir:
         manifest_filepath = Path(manifest_dir) / manifest_file
+        srcrev_filepath = Path(manifest_dir) / srcrev_file
 
         # Construct clone url containing access token
         clone_url = furl(manifest_project.http_url_to_repo)
@@ -63,22 +67,51 @@ def integrate_into_manifest(
             if name is not None and name.endswith(project.path):
                 project_node = node
         if project_node is None:
-            sys.exit("ERROR: project '%s' not found in manifest" % project.path)
-
-        # Get current project revision from manifest
-        old_revision = project_node.get("revision")
-
-        # Get new project revision from merge request
-        new_revision = merge_request.sha
-
-        # Update manifest file
-        # We are doing this using a plain text replace action. Unfortunately
-        # all python libraries for handling XML data are not able to preserve
-        # the file layout, and we want a minimal diff.
-        content = manifest_filepath.read_text()
-        content = content.replace(old_revision, new_revision)
-        manifest_filepath.write_text(content)
-        manifest_repo.index.add([manifest_file])
+            if recipe_name is None:
+                sys.exit(
+                    "ERROR: project '%s' not found in manifest and "
+                    "no recipe name is specified" % project.path
+                )
+            # Check if project is referenced in SRCREV.conf
+            project_line = None
+            content = srcrev_filepath.read_text()
+            # Match "...RECIPE_NAME ="
+            pattern = re.compile("{}[ ,\t]{{0,}}?=".format(recipe_name))
+            for line in content.splitlines():
+                if pattern.search(line):
+                    project_line = line
+            if project_line is None:
+                sys.exit(
+                    "ERROR: project '%s' not found in manifest and "
+                    "SRCREV file" % project.path
+                )
+
+            # Get current project revision from SRCREV file
+            # Assuming notation: <project> = "<hash>"
+            old_revision = project_line.split('"')[1]
+
+            # Get new project revision from merge request
+            new_revision = merge_request.sha
+
+            # Update SRCREV file
+            content = content.replace(old_revision, new_revision)
+            srcrev_filepath.write_text(content)
+            manifest_repo.index.add([srcrev_file])
+        else:
+            # Get current project revision from manifest
+            old_revision = project_node.get("revision")
+
+            # Get new project revision from merge request
+            new_revision = merge_request.sha
+
+            # Update manifest file
+            # We are doing this using a plain text replace action. Unfortunately
+            # all python libraries for handling XML data are not able to preserve
+            # the file layout, and we want a minimal diff.
+            content = manifest_filepath.read_text()
+            content = content.replace(old_revision, new_revision)
+            manifest_filepath.write_text(content)
+            manifest_repo.index.add([manifest_file])
 
         # Make an API request to create the gitlab.user object
         gitlab.auth()
@@ -134,6 +167,20 @@ def main():
         default=common.manifest_file,
         required=False,
     )
+    parser.add_argument(
+        "--srcrev-file",
+        help="""source revision file name (default: 'SRCREV.conf')""",
+        dest="srcrev_file",
+        default=common.srcrev_file,
+        required=False,
+    )
+    parser.add_argument(
+        "--recipe-name",
+        help="""recipe name to resolve project in 'SRCREV.conf'""",
+        dest="recipe_name",
+        default=None,
+        required=False,
+    )
     parser.add_argument(
         "--project",
         help="""name of the project, as specified in the manifest""",
@@ -173,6 +220,8 @@ def main():
         manifest_project=manifest_project,
         integration_base=args.integration_base,
         manifest_file=args.manifest_file,
+        srcrev_file=args.srcrev_file,
+        recipe_name=args.recipe_name,
         project=project,
         merge_request=merge_request,
     )

merge_into_manifest.py

@@ -11,7 +11,15 @@ from get_merge_requests import get_merge_requests
 from integrate_into_manifest import integrate_into_manifest
 
 
-def merge_into_manifest(manifest_project, master_branch, project, commit):
+def merge_into_manifest(
+    manifest_project,
+    master_branch,
+    project,
+    master_branch_project,
+    srcrev_file,
+    recipe_name,
+    commit,
+):
     """
     Create a merge request on the manifest for a given merged project commit and merge
     it immediately. Update the integration branch before, if it is not up to date with
@@ -21,7 +29,7 @@ def merge_into_manifest(manifest_project, master_branch, project, commit):
     # Get source merge request
     mrs = get_merge_requests(
         project,
-        target_branch=master_branch,
+        target_branch=master_branch_project,
         state="merged",
         commit=commit,
     )
@@ -75,6 +83,8 @@ def merge_into_manifest(manifest_project, master_branch, project, commit):
                 manifest_project=manifest_project,
                 integration_base=target_branch,
                 manifest_file=common.manifest_file,
+                srcrev_file=srcrev_file,
+                recipe_name=recipe_name,
                 project=project,
                 merge_request=source_mr,
             )
@@ -126,6 +136,27 @@ def main():
         dest="project",
         required=True,
     )
+    parser.add_argument(
+        "--master-branch-project",
+        help="""master branch to merge changes into (project)""",
+        dest="master_branch_project",
+        default=None,
+        required=False,
+    )
+    parser.add_argument(
+        "--srcrev-file",
+        help="""source revision file name (default: 'SRCREV.conf')""",
+        dest="srcrev_file",
+        default=common.srcrev_file,
+        required=False,
+    )
+    parser.add_argument(
+        "--recipe-name",
+        help="""recipe name to resolve the project in 'SRCREV.conf'""",
+        dest="recipe_name",
+        default=None,
+        required=False,
+    )
     parser.add_argument(
         "--commit",
         help="""sha of the commit to be merged""",
@@ -145,10 +176,18 @@ def main():
     project = common.get_project(gitlab, args.project)
     manifest_project = common.get_project(gitlab, args.manifest_project)
 
+    # If no master_branch_project is set assume manifest and project
+    # master branches are named the same
+    if not args.master_branch_project:
+        args.master_branch_project = args.master_branch
+
     manifest_revision = merge_into_manifest(
         manifest_project=manifest_project,
         master_branch=args.master_branch,
         project=project,
+        master_branch_project=args.master_branch_project,
+        srcrev_file=args.srcrev_file,
+        recipe_name=args.recipe_name,
         commit=args.commit,
     )