Use command shell to build docker images

Signed-off-by: Magnus Granberg <zorry@gentoo.org>

7 files changed, 140 insertions, 49 deletions

diff --git a/buildbot_gentoo_ci/config/builders.py b/buildbot_gentoo_ci/config/builders.py
index edcd0c1..b3ddb16 100644
--- a/buildbot_gentoo_ci/config/builders.py
+++ b/buildbot_gentoo_ci/config/builders.py
@@ -13,7 +13,7 @@ def getWorkersLocal(worker_type, workers):
                 worker_list.append(worker['name'])
         return worker_list
 
-def getWorkersDocker(worker_type, workers):
+def getWorkers(worker_type, workers):
         worker_list = []
         for worker in workers:
             if worker['type'] == worker_type and worker['enable'] is True:
@@ -44,11 +44,11 @@ def CanWorkerUpdateV(builder, wfb, request):
     print('Find a worker: NO')
     return False
 
-def gentoo_builders(worker_data_local, worker_data_docker):
+def gentoo_builders(worker_data):
     b = []
     b.append(util.BuilderConfig(
         name='update_db_check',
-        workername=getWorkersLocal('local', worker_data_local)[0],
+        workername=getWorkersLocal('local', worker_data['local'])[0],
         workerbuilddir='builds',
         collapseRequests=False,
         factory=buildfactorys.update_db_check()
@@ -56,7 +56,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
     )
     b.append(util.BuilderConfig(
         name='update_repo_check',
-        workername=getWorkersLocal('local', worker_data_local)[1],
+        workername=getWorkersLocal('local', worker_data['local'])[1],
         workerbuilddir='builds',
         collapseRequests=True,
         factory=buildfactorys.update_repo_check()
@@ -67,7 +67,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
     # (builders.UpdateRepos step)
     b.append(util.BuilderConfig(
         name='update_cpv_data',
-        workernames=getWorkersDocker('log', worker_data_docker)[0],
+        workernames=getWorkers('log', worker_data['docker'])[0],
         workerbuilddir='builds',
         collapseRequests=False,
         factory=buildfactorys.update_db_cpv()
@@ -76,7 +76,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='update_v_data',
-        workername=getWorkersDocker('log', worker_data_docker)[0],
+        workername=getWorkers('log', worker_data['docker'])[0],
         workerbuilddir='builds',
         collapseRequests=False,
         canStartBuild=CanWorkerUpdateV,
@@ -86,7 +86,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='build_request_data',
-        workernames=getWorkersLocal('local', worker_data_local),
+        workernames=getWorkersLocal('local', worker_data['local']),
         collapseRequests=False,
         factory=buildfactorys.build_request_check()
         )
@@ -94,7 +94,7 @@ def gentoo_builders(worker_data_local, worker_data_docker):
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='run_build_request',
-        workernames=getWorkersDocker('build', worker_data_docker),
+        workernames=getWorkers('build', worker_data['docker']),
         canStartBuild=CanWorkerBuildProject,
         collapseRequests=False,
         factory=buildfactorys.run_build_request()
@@ -103,19 +103,19 @@ def gentoo_builders(worker_data_local, worker_data_docker):
     # Use multiplay workers
     b.append(util.BuilderConfig(
         name='parse_build_log',
-        workernames=getWorkersDocker('log', worker_data_docker)[1:],
+        workernames=getWorkers('log', worker_data['docker'])[1:],
         collapseRequests=False,
         factory=buildfactorys.parse_build_log()
         )
     )
     # For node workers
     b.append(util.BuilderConfig(
-        name='run_build_stage4_request',
-        workernames=getWorkersLocal('local', worker_data_local),
+        name='run_build_images_request',
+        workernames=getWorkers('node', worker_data['node']),
         #FIXME: support more the one node
         #canStartBuild=CanWorkerBuildProject,
         collapseRequests=False,
-        factory=buildfactorys.run_build_stage4_request()
+        factory=buildfactorys.run_build_images_request()
         )
     )
     return b
diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index d07a143..efccc93 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -193,10 +193,10 @@ def parse_build_log():
     #f.addStep(logs.SetIrcInfo())
     return f
 
-def run_build_stage4_request():
+def run_build_images_request():
     f = util.BuildFactory()
     # set needed Propertys
     f.addStep(nodes.SetupPropertys())
-    # set the needed steps for making the stage4
-    f.addStep(nodes.SetupStage4Steps())
+    # set the needed steps for making the image
+    f.addStep(nodes.SetupBuildTypeAndSteps())
     return f
diff --git a/buildbot_gentoo_ci/config/schedulers.py b/buildbot_gentoo_ci/config/schedulers.py
index 60f1857..7833233 100644
--- a/buildbot_gentoo_ci/config/schedulers.py
+++ b/buildbot_gentoo_ci/config/schedulers.py
@@ -66,23 +66,17 @@ def gentoo_schedulers():
         builderNames = builderUpdateDbNames,
         change_filter=util.ChangeFilter(branch_fn=bb_branch_fn, category_fn=bb_category_fn),
     )
-    create_stage4 = schedulers.ForceScheduler(
-        name="create_stage4",
-        buttonName="Create stage4",
-        label="My nice create stage4 form",
-        builderNames=['run_build_stage4_request'],
+    create_images = schedulers.ForceScheduler(
+        name="create_images",
+        buttonName="Create images",
+        label="My nice create images form",
+        builderNames=['run_build_images_request'],
         # A completely customized property list.  The name of the
         # property is the name of the parameter
         properties=[
             util.StringParameter(name="project_uuid",
                     label="Project uuid",
                     default="e89c2c1a-46e0-4ded-81dd-c51afeb7fcfd", size=36),
-            util.StringParameter(name="worker",
-                    label="New worker uuid",
-                    default="a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd", size=36),
-            util.StringParameter(name="type",
-                    label="type of worker",
-                    default="docker", size=10),
         ])
     update_cpv_data = schedulers.Triggerable(name="update_cpv_data",
                                builderNames=["update_cpv_data"])
@@ -96,10 +90,10 @@ def gentoo_schedulers():
                                builderNames=["run_build_request"])
     parse_build_log = schedulers.Triggerable(name="parse_build_log",
                                builderNames=["parse_build_log"])
-    run_build_stage4_request = schedulers.Triggerable(name="run_build_stage4_request",
-                               builderNames=["run_build_stage4_request"])
+    run_build_images_request = schedulers.Triggerable(name="run_build_images_request",
+                               builderNames=["run_build_images_request"])
     s = []
-    s.append(create_stage4)
+    s.append(create_images)
     s.append(scheduler_update_db)
     s.append(update_repo_check)
     s.append(update_cpv_data)
diff --git a/buildbot_gentoo_ci/config/workers.py b/buildbot_gentoo_ci/config/workers.py
index c29502e..7d13f46 100644
--- a/buildbot_gentoo_ci/config/workers.py
+++ b/buildbot_gentoo_ci/config/workers.py
@@ -54,11 +54,15 @@ def GetBuildDockerImage(props, docker_worker):
     print(image_data)
     return image_data['tag']
 
-def gentoo_workers(worker_data_local, worker_data_docker):
+def gentoo_workers(worker_data):
     w = []
-    for local_worker in worker_data_local:
+    for local_worker in worker_data['local']:
         if local_worker['enable']:
             w.append(worker.LocalWorker(local_worker['name']))
+    # node workers
+    for node_worker in worker_data['node']:
+        if node_worker['enable']:
+            w.append(worker.Worker(node_worker['uuid'], node_worker['password']))
     # docker workers
     docker_hostconfig = {}
     # For use of sandbox stuff
@@ -67,8 +71,8 @@ def gentoo_workers(worker_data_local, worker_data_docker):
     # libseccomp overhead
     # https://github.com/seccomp/libseccomp/issues/153
     docker_hostconfig['security_opt'] = ['seccomp=unconfined']
-    print(worker_data_docker)
-    for docker_worker in worker_data_docker:
+    print(worker_data['docker'])
+    for docker_worker in worker_data['docker']:
         #FIXME: get settings from db
         if docker_worker['type'] == 'build':
             w.append(worker.DockerLatentWorker(docker_worker['uuid'],
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index a4b5935..4373ebb 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -354,6 +354,7 @@ class Model(base.DBConnectorComponent):
         sa.Column('tag', sa.String(255), nullable=False),
         sa.Column('dockerfile', sa.String(255), nullable=False),
         sa.Column('bootstrap_tag', sa.String(255), nullable=False),
+        sa.Column('type', sa.Enum('docker',), nullable=False),
     )
 
     workers_configs = sautils.Table(
diff --git a/buildbot_gentoo_ci/db/workers.py b/buildbot_gentoo_ci/db/workers.py
index d09d88c..16bc4a1 100644
--- a/buildbot_gentoo_ci/db/workers.py
+++ b/buildbot_gentoo_ci/db/workers.py
@@ -161,7 +161,8 @@ class WorkersConnectorComponent(base.DBConnectorComponent):
             name=row.name,
             tag=row.tag,
             dockerfile=row.dockerfile,
-            bootstrap_tag=row.bootstrap_tag
+            bootstrap_tag=row.bootstrap_tag,
+            type=row.type
             )
     def _row2dict_worker_config(self, conn, row):
         return dict(
diff --git a/buildbot_gentoo_ci/steps/nodes.py b/buildbot_gentoo_ci/steps/nodes.py
index 902f7a8..f92727b 100644
--- a/buildbot_gentoo_ci/steps/nodes.py
+++ b/buildbot_gentoo_ci/steps/nodes.py
@@ -25,7 +25,7 @@ from buildbot_gentoo_ci.steps import portage as portage_steps
 from buildbot_gentoo_ci.steps import repos as repos_steps
 
 class SetupPropertys(BuildStep):
-    name = 'Setup propertys for stage4 image'
+    name = 'Setup propertys for images build'
     description = 'Running'
     descriptionSuffix = None
     haltOnFailure = True
@@ -40,14 +40,112 @@ class SetupPropertys(BuildStep):
         self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
         project_data = yield self.gentooci.db.projects.getProjectByUuid(self.getProperty('project_uuid'))
         self.setProperty('project_data', project_data, 'project_data')
-        #FIXME: set it in db node config
-        self.workerbase = yield os.path.join('/', 'srv', 'gentoo', 'stage4')
-        self.workerdest = yield os.path.join(self.workerbase, self.getProperty('project_uuid'))
+        #FIXME: set it in node config
+        self.workerbase = yield os.path.join('/', 'srv', 'gentoo')
+        #self.workerdest = yield os.path.join(self.workerbase, self.getProperty('project_uuid'))
         self.setProperty('workerbase', self.workerbase, 'workerbase')
-        self.setProperty('workerdest', self.workerdest, 'workerdest')
-        self.setProperty('stage3', 'image', 'stage3')
-        # we only support docker for now
-        self.setProperty('type', 'docker', 'type')
+        #self.setProperty('workerdest', self.workerdest, 'workerdest')
+        node_info = {}
+        workers = yield self.gentooci.db.projects.getWorkersByProjectUuid(project_data['uuid'])
+        print(workers)
+        for worker in workers:
+            worker_data = yield self.gentooci.db.workers.getWorkerByUuid(worker['worker_uuid'])
+            print(worker_data)
+            node_data = yield self.gentooci.db.workers.getNodeByWorkerUuid(worker['worker_uuid'])
+            print(node_data)
+            if node_info == {} or node_info['uuid'] != node_data['node_uuid']:
+                node_info = yield self.gentooci.db.workers.getNodeByUuid(node_data['node_uuid'])
+                image_info = yield self.gentooci.db.workers.getImageById(worker_data['image_id'])
+                print(node_info)
+                print(image_info)
+                self.setProperty('node_data', node_info, 'node_data')
+                self.setProperty('image_data', image_info, 'image_data')
+        return SUCCESS
+
+class SetupBuildStepsForDocker(BuildStep):
+    name = 'Setup build steps for building docker images'
+    description = 'Running'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        node_data = self.getProperty("node_data")
+        image_data = self.getProperty("image_data")
+        dockerfile = self.getProperty("image_data")['dockerfile'] + '.Dockerfile'
+        dockerfile_path = os.path.join(self.master.basedir, 'docker')
+        #FIXME: Set build timeout in config
+        self.build_timeout = 6600
+        aftersteps_list = []
+        aftersteps_list.append(steps.FileDownload(
+                        flunkOnFailure=True,
+                        name='Upload Dockerfile',
+                        mastersrc = os.path.join(dockerfile_path, dockerfile),
+                        workerdest = dockerfile
+                    ))
+        # build the gentoo docker buildbot-worker image
+        # set the build args
+        self.buildargs = {}
+        # set PROJECTUUID
+        self.buildargs['PROJECTUUID'] = self.getProperty("project_uuid")
+        # set ACCEPT_KEYWORDS
+        keyword_data = yield self.gentooci.db.keywords.getKeywordById(self.getProperty('project_data')['keyword_id'])
+        if self.getProperty('project_data')['status'] == 'unstable':
+            self.buildargs['KEYWORDS'] = '~' + keyword_data['name']
+        else:
+            self.buildargs['KEYWORDS'] = keyword_data['name']
+        # set ARCH
+        self.buildargs['ARCH'] = keyword_data['name']
+        # set MAKEOPTS
+        # set USE
+        # set CFLAGS
+        # set BINHOSTURL
+        self.buildargs['BINHOSTURL'] = node_data['bin_host_url']
+        # set bootstrap image
+        self.buildargs['BOOTSTRAPTAG'] = image_data['bootstrap_tag']
+        # build the gentoo docker buildbot-worker image
+        aftersteps_list.append(steps.ShellCommand(
+                        flunkOnFailure=True,
+                        name='Build docker buildbot-worker image',
+                        command=['docker',
+                                 'buildx',
+                                 'build',
+                                 '--file', dockerfile,
+                                 '--build-arg', 'PROJECTUUID='+ self.buildargs['PROJECTUUID'],
+                                 '--build-arg', 'KEYWORDS=' + self.buildargs['KEYWORDS'],
+                                 '--build-arg', 'ARCH=' + self.buildargs['ARCH'],
+                                 '--build-arg', 'BINHOSTURL=' + self.buildargs['BINHOSTURL'],
+                                 '--build-arg', 'BOOTSTRAPTAG=' + self.buildargs['BOOTSTRAPTAG'],
+                                 '--tag', image_data['tag'],
+                                 '.'],
+                        timeout=self.build_timeout
+                    ))
+        if aftersteps_list != []:
+            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+        return SUCCESS
+
+class SetupBuildTypeAndSteps(BuildStep):
+    name = 'Check build type for building images'
+    description = 'Running'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @defer.inlineCallbacks
+    def run(self):
+        aftersteps_list = []
+        if self.getProperty("image_data")['type'] == 'docker':
+            aftersteps_list.append(SetupBuildStepsForDocker())
+        if aftersteps_list != []:
+            yield self.build.addStepsAfterCurrentStep(aftersteps_list)
         return SUCCESS
 
 class SetupStage4Steps(BuildStep):
@@ -204,13 +302,6 @@ class SetupStage4Steps(BuildStep):
                         command=['docker', 'import', 'stage4-' + self.getProperty('project_uuid') + '.tar', 'stage4-' + self.getProperty('project_uuid') + ':latest'],
                         workdir=self.getProperty("workerbase")
                     ))
-                    # gentoo docker buildbot-worker image
-                    aftersteps_list.append(steps.ShellCommand(
-                        flunkOnFailure=True,
-                        name='Build docker buildbot-worker image',
-                        command=['docker', 'buildx', 'build', '--file', '../docker/GentooBuildbotWorker.Docker', '--build-arg', 'PROJECTUUID=' + self.getProperty('project_uuid'), '--tag', 'bb-worker-' +  self.getProperty('project_uuid') + ':latest', '.'],
-                        workdir=self.getProperty("workerbase")
-                    ))
         if aftersteps_list != []:
             yield self.build.addStepsAfterCurrentStep(aftersteps_list)
         return SUCCESS