| From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
| From: Tristan Cacqueray <tdecacqu@redhat.com> |
| Date: Sat, 1 Dec 2018 07:58:51 +0000 |
| Subject: [PATCH] config: add tenant.toDict() method and REST endpoint |
| |
| This change adds a new /config endpoint to introspect a zuul tenant |
| configuration. The new endpoint can be used to get the global configuration in |
| one request instead of having to query each individual endpoints, for example to |
| check which projects use which jobs or nodesets. |
| |
| Change-Id: I5d3c22b205a5228354a51eb1fe2f1c900cf455d2 |
| --- |
| zuul/model.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| zuul/rpclistener.py | 30 ++++++++++-------------- |
| zuul/web/__init__.py | 29 +++++++++++++++++++++++ |
| 3 files changed, 107 insertions(+), 18 deletions(-) |
| |
| diff --git a/zuul/model.py b/zuul/model.py |
| index 30739a1..ba3d572 100644 |
| --- a/zuul/model.py |
| +++ b/zuul/model.py |
| @@ -3649,6 +3649,31 @@ class Layout(object): |
| self.semaphores = {} |
| self.loading_errors = LoadingErrors() |
| |
| + def toDict(self): |
| + d = { |
| + 'pipelines': [], |
| + 'jobs': [], |
| + 'nodesets': [], |
| + 'secrets': [], |
| + 'semaphores': [], |
| + } |
| + for pipeline in self.pipelines.keys(): |
| + d['pipelines'].append({'name': pipeline}) |
| + for job_name in sorted(self.jobs): |
| + if job_name == "noop": |
| + continue |
| + dj = [] |
| + for job in self.jobs[job_name]: |
| + dj.append(job.toDict(self.tenant)) |
| + d['jobs'].append(dj) |
| + for nodeset in sorted(self.nodesets): |
| + d['nodesets'].append(self.nodesets[nodeset].toDict()) |
| + for secret in sorted(self.secrets): |
| + d['secrets'].append({'name': secret}) |
| + for semaphore in sorted(self.semaphores): |
| + d['semaphores'].append({'name': semaphore}) |
| + return d |
| + |
| def getJob(self, name): |
| if name in self.jobs: |
| return self.jobs[name][0] |
| @@ -3818,6 +3843,28 @@ class Layout(object): |
| self.log.warning("%s for project %s" % (e, name)) |
| return [] |
| |
| + def getAllProjectConfigsJson(self, name): |
| + ret = [] |
| + configs = self.getAllProjectConfigs(name) |
| + for config_obj in configs: |
| + config = config_obj.toDict() |
| + config['pipelines'] = [] |
| + for pipeline_name, pipeline_config in sorted( |
| + config_obj.pipelines.items()): |
| + pipeline = pipeline_config.toDict() |
| + pipeline['name'] = pipeline_name |
| + pipeline['jobs'] = [] |
| + for job_name, jobs in pipeline_config.job_list.jobs.items(): |
| + if job_name == "noop": |
| + continue |
| + job_list = [] |
| + for job in jobs: |
| + job_list.append(job.toDict(self.tenant)) |
| + pipeline['jobs'].append(job_list) |
| + config['pipelines'].append(pipeline) |
| + ret.append(config) |
| + return ret |
| + |
| def getProjectMetadata(self, name): |
| if name in self.project_metadata: |
| return self.project_metadata[name] |
| @@ -4275,6 +4322,25 @@ class Tenant(object): |
| hostname_dict[project.canonical_hostname] = project |
| self.project_configs[project.canonical_name] = tpc |
| |
| + def toDict(self): |
| + d = { |
| + 'name': self.name, |
| + 'projects': [], |
| + 'layout': self.layout.toDict(), |
| + } |
| + for project in self.config_projects: |
| + dp = project.toDict() |
| + dp['type'] = "config" |
| + d['projects'].append(dp) |
| + for project in self.untrusted_projects: |
| + dp = project.toDict() |
| + dp['type'] = "untrusted" |
| + d['projects'].append(dp) |
| + for project in d['projects']: |
| + project['configs'] = self.layout.getAllProjectConfigsJson( |
| + project['canonical_name']) |
| + return d |
| + |
| def getProject(self, name): |
| """Return a project given its name. |
| |
| diff --git a/zuul/rpclistener.py b/zuul/rpclistener.py |
| index 2e14413..0936353 100644 |
| --- a/zuul/rpclistener.py |
| +++ b/zuul/rpclistener.py |
| @@ -52,6 +52,7 @@ class RPCListener(object): |
| 'project_freeze_jobs', |
| 'pipeline_list', |
| 'key_get', |
| + 'config_get', |
| 'config_errors_list', |
| 'connection_list', |
| 'authorize_user', |
| @@ -397,24 +398,8 @@ class RPCListener(object): |
| gear_job.sendWorkComplete(json.dumps({})) |
| return |
| result = project.toDict() |
| - result['configs'] = [] |
| - configs = tenant.layout.getAllProjectConfigs(project.canonical_name) |
| - for config_obj in configs: |
| - config = config_obj.toDict() |
| - config['pipelines'] = [] |
| - for pipeline_name, pipeline_config in sorted( |
| - config_obj.pipelines.items()): |
| - pipeline = pipeline_config.toDict() |
| - pipeline['name'] = pipeline_name |
| - pipeline['jobs'] = [] |
| - for jobs in pipeline_config.job_list.jobs.values(): |
| - job_list = [] |
| - for job in jobs: |
| - job_list.append(job.toDict(tenant)) |
| - pipeline['jobs'].append(job_list) |
| - config['pipelines'].append(pipeline) |
| - result['configs'].append(config) |
| - |
| + result['configs'] = tenant.layout.getAllProjectConfigsJson( |
| + project.canonical_name) |
| gear_job.sendWorkComplete(json.dumps(result, cls=ZuulJSONEncoder)) |
| |
| def handle_project_list(self, job): |
| @@ -537,3 +522,12 @@ class RPCListener(object): |
| for source in self.sched.connections.getSources(): |
| output.append(source.connection.toDict()) |
| job.sendWorkComplete(json.dumps(output)) |
| + |
| + def handle_config_get(self, job): |
| + args = json.loads(job.arguments) |
| + tenant = self.sched.abide.tenants.get(args.get("tenant")) |
| + if not tenant: |
| + job.sendWorkComplete(json.dumps(None)) |
| + return |
| + job.sendWorkComplete( |
| + json.dumps(tenant.toDict(), cls=ZuulJSONEncoder)) |
| diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py |
| index 4f6ce13..c72d368 100755 |
| --- a/zuul/web/__init__.py |
| +++ b/zuul/web/__init__.py |
| @@ -212,6 +212,8 @@ class ZuulWebAPI(object): |
| self.cache_expiry = 1 |
| self.static_cache_expiry = zuulweb.static_cache_expiry |
| self.status_lock = threading.Lock() |
| + self.config_cache = {} |
| + self.config_lock = threading.Lock() |
| |
| def _basic_auth_header_check(self): |
| """make sure protected endpoints have a Authorization header with the |
| @@ -581,6 +583,31 @@ class ZuulWebAPI(object): |
| @cherrypy.expose |
| @cherrypy.tools.save_params() |
| @cherrypy.tools.json_out(content_type='application/json; charset=utf-8') |
| + def config(self, tenant): |
| + now = time.time() |
| + with self.config_lock: |
| + if tenant not in self.config_cache or \ |
| + now > self.config_cache[tenant]["ttl"]: |
| + data = self.rpc.submitJob( |
| + 'zuul:config_get', {'tenant': tenant}).data[0] |
| + config = json.loads(data) |
| + if config is None: |
| + raise cherrypy.HTTPError( |
| + 404, 'Tenant %s does not exist.' % tenant) |
| + config["generatedAt"] = int(now) |
| + self.config_cache[tenant] = { |
| + "config": config, |
| + # Expire after 1 hour per MB, up to one day |
| + "ttl": now + min(3600 * 24, 3600 * len(data) / 2 ** 20) |
| + } |
| + ret = self.config_cache[tenant]["config"] |
| + resp = cherrypy.response |
| + resp.headers['Access-Control-Allow-Origin'] = '*' |
| + return ret |
| + |
| + @cherrypy.expose |
| + @cherrypy.tools.save_params() |
| + @cherrypy.tools.json_out(content_type='application/json; charset=utf-8') |
| def job(self, tenant, job_name): |
| job = self.rpc.submitJob( |
| 'zuul:job_get', {'tenant': tenant, 'job': job_name}) |
| @@ -1072,6 +1099,8 @@ class ZuulWeb(object): |
| controller=api, action='buildset') |
| route_map.connect('api', '/api/tenant/{tenant}/config-errors', |
| controller=api, action='config_errors') |
| + route_map.connect('api', '/api/tenant/{tenant}/config', |
| + controller=api, action='config') |
| |
| for connection in connections.connections.values(): |
| controller = connection.getWebController(self) |
| -- |
| 1.8.3.1 |
| |