summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSF initial configurator <admin@softwarefactory-project.io>2019-12-16 22:19:21 +0000
committerSF initial configurator <admin@softwarefactory-project.io>2019-12-16 22:19:21 +0000
commite9c0c35c95b0dfce367a7031439e5d60791c60da (patch)
tree658ad5ecb13d49aad0ff1c0c08b6e726fe92f34b
parent25572ef0f33e717478eb74c7eb1bc4ce165ea1af (diff)
Automatic update of defaults
Change-Id: I494ff97fb21fb2747e237d0062434b8b0c825ed3
-rw-r--r--playbooks/releasenotes/post.yaml4
-rw-r--r--playbooks/tox/docs-post.yaml2
-rw-r--r--roles/ara-report/tasks/main.yaml15
-rw-r--r--roles/emit-job-header/README.rst10
-rw-r--r--roles/fetch-sphinx-output/tasks/main.yaml10
-rw-r--r--roles/fetch-sphinx-tarball/README.rst7
-rw-r--r--roles/fetch-sphinx-tarball/defaults/main.yaml2
-rw-r--r--roles/fetch-sphinx-tarball/tasks/main.yaml4
-rw-r--r--roles/fetch-sphinx-tarball/tasks/pdf.yaml56
-rw-r--r--roles/generate-zuul-manifest/library/generate_manifest.py15
-rw-r--r--roles/generate-zuul-manifest/library/test_generate_manifest.py17
-rw-r--r--roles/multi-node-bridge/tasks/peer.yaml23
-rw-r--r--roles/multi-node-bridge/templates/zuul-multi-node-bridge-ovs.repo.j26
-rw-r--r--roles/multi-node-bridge/vars/RedHat.yaml7
-rw-r--r--roles/prepare-imagestream-openshift/README.rst11
-rw-r--r--roles/prepare-imagestream-openshift/tasks/main.yaml175
-rw-r--r--roles/set-zuul-log-path-fact/README.rst11
-rw-r--r--roles/set-zuul-log-path-fact/defaults/main.yaml1
-rw-r--r--roles/set-zuul-log-path-fact/tasks/main.yaml49
-rw-r--r--roles/tox/library/tox_install_sibling_packages.py4
-rw-r--r--roles/upload-logs-swift/README.rst16
-rw-r--r--roles/upload-logs-swift/library/test_zuul_swift_upload.py25
-rwxr-xr-xroles/upload-logs-swift/library/zuul_swift_upload.py150
-rw-r--r--roles/upload-logs/README.rst11
-rw-r--r--zuul.d/js-jobs.yaml2
-rw-r--r--zuul.d/python-jobs.yaml4
26 files changed, 356 insertions, 281 deletions
diff --git a/playbooks/releasenotes/post.yaml b/playbooks/releasenotes/post.yaml
index 5176176..71c8660 100644
--- a/playbooks/releasenotes/post.yaml
+++ b/playbooks/releasenotes/post.yaml
@@ -1,4 +1,4 @@
- hosts: all
roles:
- - role: fetch-sphinx-output
- sphinx_output_src: "{{ zuul_work_dir|default(zuul.project.src_dir) }}/releasenotes/build/html"
+ - role: fetch-sphinx-tarball
+ sphinx_build_dir: "releasenotes/build"
diff --git a/playbooks/tox/docs-post.yaml b/playbooks/tox/docs-post.yaml
index c99e1e6..b6f4eca 100644
--- a/playbooks/tox/docs-post.yaml
+++ b/playbooks/tox/docs-post.yaml
@@ -1,3 +1,3 @@
- hosts: all
roles:
- - fetch-sphinx-output
+ - fetch-sphinx-tarball
diff --git a/roles/ara-report/tasks/main.yaml b/roles/ara-report/tasks/main.yaml
index d4bf38f..4d708a4 100644
--- a/roles/ara-report/tasks/main.yaml
+++ b/roles/ara-report/tasks/main.yaml
@@ -60,6 +60,14 @@
when:
- ara_compress_html | bool
- not ara_generated | skipped
+ - name: Return ARA report
+ when: not ara_generated | skipped
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ - name: ARA report
+ url: "{{ ara_report_path }}"
rescue:
- name: HTML generation rescue
debug:
@@ -80,3 +88,10 @@
- name: Save the ARA database
command: cp {{ ara_database_path }} {{ final_ara_report_path }}
+ - name: Return ARA report
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ - name: ARA report
+ url: "{{ ara_report_path }}"
diff --git a/roles/emit-job-header/README.rst b/roles/emit-job-header/README.rst
index bdb97c7..b67d66b 100644
--- a/roles/emit-job-header/README.rst
+++ b/roles/emit-job-header/README.rst
@@ -5,3 +5,13 @@ Log a few lines about the job.
.. zuul:rolevar:: zuul_log_url
Base URL where logs are to be found.
+
+.. zuul:rolevar:: zuul_log_path_shard_build
+ :default: False
+
+ This var is consumed by set-zuul-log-path-fact which emit-job-header
+ calls into. If you set this you will get log paths prefixed with the
+ first three characters of the build uuid. This will improve log file
+ sharding.
+
+ More details can be found at :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`
diff --git a/roles/fetch-sphinx-output/tasks/main.yaml b/roles/fetch-sphinx-output/tasks/main.yaml
index 6152799..d545b01 100644
--- a/roles/fetch-sphinx-output/tasks/main.yaml
+++ b/roles/fetch-sphinx-output/tasks/main.yaml
@@ -4,3 +4,13 @@
mode: pull
src: "{{ sphinx_output_src }}"
verify_host: true
+
+- name: Return artifact to Zuul
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ - name: "Docs preview site"
+ url: "html/"
+ metadata:
+ type: docs_site
diff --git a/roles/fetch-sphinx-tarball/README.rst b/roles/fetch-sphinx-tarball/README.rst
index a1af64c..d78cf9f 100644
--- a/roles/fetch-sphinx-tarball/README.rst
+++ b/roles/fetch-sphinx-tarball/README.rst
@@ -16,3 +16,10 @@ archive into the log root for viewing.
:default: {{ zuul.project.src_dir }}
The location of the main working directory of the job.
+
+.. zuul:rolevar:: sphinx_pdf_files
+ :default: list
+
+ A list of file names of PDF files to collect.
+ By default, the list contains as entry only
+ ``doc-{{ zuul.project.short_name }}.pdf``.
diff --git a/roles/fetch-sphinx-tarball/defaults/main.yaml b/roles/fetch-sphinx-tarball/defaults/main.yaml
index 4c68b38..a49d3c3 100644
--- a/roles/fetch-sphinx-tarball/defaults/main.yaml
+++ b/roles/fetch-sphinx-tarball/defaults/main.yaml
@@ -1,3 +1,5 @@
---
zuul_work_dir: "{{ zuul.project.src_dir }}"
sphinx_build_dir: doc/build
+sphinx_pdf_files:
+ - "doc-{{ zuul.project.short_name }}.pdf"
diff --git a/roles/fetch-sphinx-tarball/tasks/main.yaml b/roles/fetch-sphinx-tarball/tasks/main.yaml
index 982f8c8..8c23851 100644
--- a/roles/fetch-sphinx-tarball/tasks/main.yaml
+++ b/roles/fetch-sphinx-tarball/tasks/main.yaml
@@ -12,4 +12,8 @@
when: "'html' in sphinx_dir"
include_tasks: html.yaml
+- name: Process sphinx PDF
+ when: "'pdf' in sphinx_dir"
+ include_tasks: pdf.yaml
+
# Other sphinx output processing can be added here.
diff --git a/roles/fetch-sphinx-tarball/tasks/pdf.yaml b/roles/fetch-sphinx-tarball/tasks/pdf.yaml
new file mode 100644
index 0000000..59edd13
--- /dev/null
+++ b/roles/fetch-sphinx-tarball/tasks/pdf.yaml
@@ -0,0 +1,56 @@
+# Sphinx might build multiple PDF files, for example for graphic files
+# to include. We only want to grab the end result and not any such
+# input files.
+
+- name: Check for PDF file names
+ stat:
+ path: "{{ zuul_work_dir }}/{{ sphinx_build_dir }}/pdf/{{ item }}"
+ get_checksum: false
+ get_mime: false
+ get_md5: false
+ with_items: "{{ sphinx_pdf_files }}"
+ register: pdf_file_stat
+
+- name: Set pdf_files_found to default
+ set_fact:
+ pdf_files_found: false
+
+- name: Check if any file found
+ set_fact:
+ pdf_files_found: true
+ when: item.stat.exists
+ with_items: "{{ pdf_file_stat.results }}"
+
+# Now loop...
+
+- name: Grab PDF files
+ when: pdf_files_found
+ block:
+
+ - name: Create PDF directory
+ delegate_to: localhost
+ file:
+ path: "{{ zuul.executor.log_root }}/pdf"
+ state: directory
+
+ - name: Fetch PDF files
+ synchronize:
+ dest: "{{ zuul.executor.log_root }}/pdf/{{ item.item }}"
+ mode: pull
+ src: "{{ item.stat.path }}"
+ verify_host: true
+ with_items: "{{ pdf_file_stat.results }}"
+ when: item.stat.exists
+
+
+ - name: Return PDF artifact to Zuul
+ zuul_return:
+ data:
+ zuul:
+ artifacts:
+ - name: "Docs PDF: {{ item.item }}"
+ url: "pdf/{{ item.item }}"
+ metadata:
+ type: docs_pdf
+ with_items: "{{ pdf_file_stat.results }}"
+ when: item.stat.exists
diff --git a/roles/generate-zuul-manifest/library/generate_manifest.py b/roles/generate-zuul-manifest/library/generate_manifest.py
index 378607f..5518e53 100644
--- a/roles/generate-zuul-manifest/library/generate_manifest.py
+++ b/roles/generate-zuul-manifest/library/generate_manifest.py
@@ -37,6 +37,15 @@ def path_in_tree(root, path):
return True
+def _get_file_info(path):
+ try:
+ st = os.stat(path)
+ except OSError:
+ return 0, 0
+
+ return st[stat.ST_MTIME], st[stat.ST_SIZE]
+
+
def walk(root, original_root=None):
if original_root is None:
original_root = root
@@ -67,9 +76,9 @@ def walk(root, original_root=None):
mime_guess, encoding = mimetypes.guess_type(path)
if not mime_guess:
mime_guess = 'text/plain'
- st = os.stat(path)
- last_modified = st[stat.ST_MTIME]
- size = st[stat.ST_SIZE]
+ last_modified, size = _get_file_info(path)
+ if not last_modified and not size:
+ continue
data.append(dict(name=f,
mimetype=mime_guess,
encoding=encoding,
diff --git a/roles/generate-zuul-manifest/library/test_generate_manifest.py b/roles/generate-zuul-manifest/library/test_generate_manifest.py
index 99acdde..a239523 100644
--- a/roles/generate-zuul-manifest/library/test_generate_manifest.py
+++ b/roles/generate-zuul-manifest/library/test_generate_manifest.py
@@ -19,9 +19,11 @@ from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
import os
+import stat
import testtools
import fixtures
+from .generate_manifest import _get_file_info
from .generate_manifest import walk
@@ -126,3 +128,18 @@ class TestFileList(testtools.TestCase):
('controller/service_log.txt', 'text/plain', None),
('symlink_loop/placeholder', 'text/plain', None),
])
+
+ def test_get_file_info(self):
+ '''Test files info'''
+ path = os.path.join(FIXTURE_DIR, 'logs', 'job-output.json')
+ last_modified, size = _get_file_info(path)
+
+ self.assertEqual(os.stat(path)[stat.ST_MTIME], last_modified)
+ self.assertEqual(16, size)
+
+ def test_get_file_info_missing_file(self):
+ '''Test files that go missing during a walk'''
+ last_modified, size = _get_file_info('missing/file/that/we/cant/find')
+
+ self.assertEqual(0, last_modified)
+ self.assertEqual(0, size)
diff --git a/roles/multi-node-bridge/tasks/peer.yaml b/roles/multi-node-bridge/tasks/peer.yaml
index c1dea7d..8fd4182 100644
--- a/roles/multi-node-bridge/tasks/peer.yaml
+++ b/roles/multi-node-bridge/tasks/peer.yaml
@@ -9,20 +9,31 @@
vni: "{{ offset | int + bridge_vni_offset | int }}"
# To make things more readable in the following tasks
-- name: Set ip address when the node private IP is not set
+- name: Set ip address when the node private IP looks empty
+ set_fact:
+ nodepool_ip: "{{ nodepool.public_ipv4 }}"
+ when: not (nodepool.private_ipv4 | ipv4)
+
+- name: Set ip address when the node private IP was not defined
set_fact:
nodepool_ip: |
{{ nodepool.private_ipv4 | default(nodepool.public_ipv4) }}
+ when: nodepool_ip is not defined
-- name: Select the switch from group
+- name: Select the switch from group and the private ip
set_fact:
switch: "{{ groups['switch'][0] }}"
+ switch_private_ip: "{{ hostvars[groups['switch'][0]].nodepool.private_ipv4 }}"
+
+- name: Alias the primary node private IP, if it looks empty
+ set_fact:
+ switch_ip: "{{ hostvars[switch].nodepool.public_ipv4 }}"
+ when: not (switch_private_ip | ipv4)
-- name: Alias the primary node private IP
+- name: Alias the primary node private IP, if it was not defined
set_fact:
- switch_ip: |
- {{ hostvars[switch].nodepool.private_ipv4 |
- default(hostvars[switch].nodepool.public_ipv4) }}
+ switch_ip: "{{ switch_private_ip | default(hostvars[switch].nodepool.public_ipv4) }}"
+ when: switch_ip is not defined
- name: Add port to bridge on switch node
become: yes
diff --git a/roles/multi-node-bridge/templates/zuul-multi-node-bridge-ovs.repo.j2 b/roles/multi-node-bridge/templates/zuul-multi-node-bridge-ovs.repo.j2
index 3621474..796111b 100644
--- a/roles/multi-node-bridge/templates/zuul-multi-node-bridge-ovs.repo.j2
+++ b/roles/multi-node-bridge/templates/zuul-multi-node-bridge-ovs.repo.j2
@@ -1,5 +1,5 @@
# Vendored from rdo-release: https://github.com/rdo-infra/rdo-release
-{% if ansible_distribution == 'CentOS' %}
+{% if ansible_distribution in ['CentOS', 'RedHat'] and ansible_distribution_major_version|int <= 7 %}
[centos-openstack-queens]
name=CentOS OpenStack Queens Repository
{% if zuul_site_mirror_fqdn is defined %}
@@ -10,10 +10,10 @@ baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-queens/
gpgcheck=1
enabled=1
gpgkey=file:///tmp/RPM-GPG-KEY-CentOS-SIG-Cloud
-{% elif ansible_distribution == 'RedHat' %}
+{% elif ansible_distribution == 'RedHat' and ansible_distribution_major_version|int >= 8 %}
[RDO-RHEL8-deps]
name=RedHat Openstack deps repo
baseurl=https://trunk.rdoproject.org/rhel8-master/deps/latest/
gpgcheck=0
enabled=1
-{% endif %} \ No newline at end of file
+{% endif %}
diff --git a/roles/multi-node-bridge/vars/RedHat.yaml b/roles/multi-node-bridge/vars/RedHat.yaml
index 1ae946d..6f26a79 100644
--- a/roles/multi-node-bridge/vars/RedHat.yaml
+++ b/roles/multi-node-bridge/vars/RedHat.yaml
@@ -1,3 +1,8 @@
---
-ovs_package: "rhosp-openvswitch"
+ovs_package: >-
+ {% if ansible_distribution_major_version|int >= 8 -%}
+ rhosp-openvswitch
+ {%- else -%}
+ openvswitch
+ {%- endif %}
ovs_service: "openvswitch"
diff --git a/roles/prepare-imagestream-openshift/README.rst b/roles/prepare-imagestream-openshift/README.rst
deleted file mode 100644
index 98a3137..0000000
--- a/roles/prepare-imagestream-openshift/README.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-Prepare an openshift ImageStream
-
-This role is intended to run before BuildConfigs.
-It creates...
-
-**Role Variables**
-
-.. zuul:rolevar:: openshift_pods
- :required: true
-
- The list of openshift pods to copy the source to.
diff --git a/roles/prepare-imagestream-openshift/tasks/main.yaml b/roles/prepare-imagestream-openshift/tasks/main.yaml
deleted file mode 100644
index ab91489..0000000
--- a/roles/prepare-imagestream-openshift/tasks/main.yaml
+++ /dev/null
@@ -1,175 +0,0 @@
-#########################
-# Prepare the namespace #
-#########################
-# Note: this can probably be simplified by running a ready to use http server.
-# The http server is constructed using the openshift/httpd-ex template
-- name: create staging-http ImageStream
- openshift_raw:
- state: present
- namespace: "{{ zuul.resources[zuul_resource].namespace }}"
- definition:
- apiVersion: v1
- kind: ImageStream
- metadata:
- labels:
- app: staging-http-server
- name: staging-http-server
- spec:
- lookupPolicy:
- local: false
-
-- name: create staging-http BuildConfig
- openshift_raw:
- state: present
- namespace: "{{ zuul.resources[zuul_resource].namespace }}"
- definition:
- apiVersion: v1
- kind: BuildConfig
- metadata:
- labels:
- app: staging-http-server
- name: staging-http-server
- spec:
- output:
- to:
- kind: ImageStreamTag
- name: 'staging-http-server:latest'
- postCommit: {}
- resources: {}
- runPolicy: Serial
- source:
- git:
- ref: master
- uri: 'https://github.com/openshift/httpd-ex.git'
- type: Git
- strategy:
- sourceStrategy:
- from:
- kind: ImageStreamTag
- name: 'httpd:2.4'
- namespace: openshift
- type: Source
- triggers:
- - type: ImageChange
- - type: ConfigChange
- status:
- lastVersion: 1
-
-- name: create staging-http DeploymentConfig
- openshift_raw:
- state: present
- namespace: "{{ zuul.resources[zuul_resource].namespace }}"
- definition:
- apiVersion: v1
- kind: DeploymentConfig
- metadata:
- generation: 2
- labels:
- app: staging-http-server
- name: staging-http-server
- spec:
- replicas: 1
- selector:
- deploymentconfig: staging-http-server
- strategy:
- resources: {}
- type: Rolling
- template:
- metadata:
- labels:
- app: staging-http-server
- deploymentconfig: staging-http-server
- spec:
- containers:
- - image: "172.30.1.1:5000/{{ zuul.resources[zuul_resource].namespace }}/staging-http-server"
- # imagePullPolicy: Always
- name: staging-http-server
- ports:
- - containerPort: 8080
- protocol: TCP
- - containerPort: 8443
- protocol: TCP
- resources: {}
- dnsPolicy: ClusterFirst
- restartPolicy: Always
- schedulerName: default-scheduler
- securityContext: {}
- terminationGracePeriodSeconds: 30
- test: false
-
-- name: create staging-http Service spec
- openshift_raw:
- state: present
- namespace: "{{ zuul.resources[zuul_resource].namespace }}"
- definition:
- apiVersion: v1
- kind: Service
- metadata:
- labels:
- app: staging-http-server
- name: staging-http-server
- spec:
- ports:
- - name: 8080-tcp
- port: 8080
- protocol: TCP
- targetPort: 8080
- selector:
- deploymentconfig: staging-http-server
- sessionAffinity: None
- type: ClusterIP
- status:
- loadBalancer: {}
-
-- name: get staging-http-server pod name
- command: oc get pods --field-selector=status.phase=Running -o "jsonpath={.items[?(@.metadata.labels.app=='staging-http-server')].metadata.name}"
- register: _zm_name
- retries: 600
- delay: 1
- until: "'staging-http' in _zm_name.stdout"
-
-- name: register staging-http-server pod name
- set_fact:
- zm_name: "{{ _zm_name.stdout }}"
-
-###########################
-# Build the project image #
-###########################
-- name: prepare dumb bare clone of future state
- git:
- repo: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}"
- dest: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}.git"
- bare: yes
- tags:
- # We don't specify git version to re-use executor state
- - skip_ansible_lint
-
-- name: update server info for dumb http transport
- command: git update-server-info
- args:
- chdir: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}.git"
- tags:
- # Git module doesn't support update-server-info command
- - skip_ansible_lint
-
-- name: create project dir on http server
- command: "oc exec {{ zm_name }} -- mkdir -p {{ zuul.project.src_dir }}.git"
-
-- name: copy project to http server
- command: "oc rsync {{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}.git/ {{ zm_name }}:/opt/app-root/src/{{ zuul.project.src_dir }}.git/"
-
-- name: create project ImageStream spec
- openshift_raw:
- state: present
- namespace: "{{ zuul.resources['openshift-project'].namespace }}"
- definition:
- apiVersion: v1
- kind: ImageStream
- metadata:
- generation: 1
- labels:
- app: "{{ zuul.project.short_name }}"
- name: "{{ zuul.project.short_name }}"
- spec:
- lookupPolicy:
- local: false
diff --git a/roles/set-zuul-log-path-fact/README.rst b/roles/set-zuul-log-path-fact/README.rst
index d64eec8..33a36d1 100644
--- a/roles/set-zuul-log-path-fact/README.rst
+++ b/roles/set-zuul-log-path-fact/README.rst
@@ -1 +1,12 @@
Sets a fact named ``zuul_log_path`` from zuul variables
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_log_path_shard_build
+ :type: bool
+ :default: False
+
+ Flag to specify whether or not paths that include a three character
+ prefix based on the build uuid should prefix the log path. This is
+ particularly useful for object storage systems where we want to
+ spread out the number of files per container.
diff --git a/roles/set-zuul-log-path-fact/defaults/main.yaml b/roles/set-zuul-log-path-fact/defaults/main.yaml
new file mode 100644
index 0000000..b1abb3a
--- /dev/null
+++ b/roles/set-zuul-log-path-fact/defaults/main.yaml
@@ -0,0 +1 @@
+zuul_log_path_shard_build: false
diff --git a/roles/set-zuul-log-path-fact/tasks/main.yaml b/roles/set-zuul-log-path-fact/tasks/main.yaml
index c49c3a8..f9dff1b 100644
--- a/roles/set-zuul-log-path-fact/tasks/main.yaml
+++ b/roles/set-zuul-log-path-fact/tasks/main.yaml
@@ -1,14 +1,35 @@
-- name: Set log path for a change
- when: zuul.change is defined
- set_fact:
- zuul_log_path: "{{ zuul.change[-2:] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
-
-- name: Set log path for a ref update
- when: zuul.newrev is defined
- set_fact:
- zuul_log_path: "{{ zuul.newrev[:2] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
-
-- name: Set log path for a periodic job
- when: zuul.change is not defined and zuul.newrev is not defined
- set_fact:
- zuul_log_path: "{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+- name: Fileserver friendly log path specifications
+ when: not zuul_log_path_shard_build
+ block:
+ - name: Set log path for a change
+ when: zuul.change is defined
+ set_fact:
+ zuul_log_path: "{{ zuul.change[-2:] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+
+ - name: Set log path for a ref update
+ when: zuul.newrev is defined
+ set_fact:
+ zuul_log_path: "{{ zuul.newrev[:2] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+
+ - name: Set log path for a periodic job
+ when: zuul.change is not defined and zuul.newrev is not defined
+ set_fact:
+ zuul_log_path: "{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+
+- name: object store friendly log path specifications
+ when: zuul_log_path_shard_build
+ block:
+ - name: Set log path for a change
+ when: zuul.change is defined
+ set_fact:
+ zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+
+ - name: Set log path for a ref update
+ when: zuul.newrev is defined
+ set_fact:
+ zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+
+ - name: Set log path for a periodic job
+ when: zuul.change is not defined and zuul.newrev is not defined
+ set_fact:
+ zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
diff --git a/roles/tox/library/tox_install_sibling_packages.py b/roles/tox/library/tox_install_sibling_packages.py
index abec207..d2a0410 100644
--- a/roles/tox/library/tox_install_sibling_packages.py
+++ b/roles/tox/library/tox_install_sibling_packages.py
@@ -95,7 +95,7 @@ def get_sibling_python_packages(projects, tox_python):
# package name is.
package_name = subprocess.check_output(
[os.path.abspath(tox_python), 'setup.py', '--name'],
- cwd=os.path.abspath(root)).decode('utf-8')
+ cwd=os.path.abspath(root))
if package_name:
package_name = package_name.strip()
packages[package_name] = root
@@ -111,7 +111,7 @@ def get_installed_packages(tox_python):
# interface.
frozen_pkgs = subprocess.check_output(
[tox_python, '-m', 'pip', '-qqq', 'freeze']
- ).decode('utf-8')
+ )
# Matches strings of the form:
# 1. '<package_name>==<version>'
# 2. '# Editable Git install with no remote (<package_name>==<version>)'
diff --git a/roles/upload-logs-swift/README.rst b/roles/upload-logs-swift/README.rst
index 17c661e..fb21c7b 100644
--- a/roles/upload-logs-swift/README.rst
+++ b/roles/upload-logs-swift/README.rst
@@ -37,6 +37,11 @@ This uploads logs to an OpenStack Object Store (Swift) container.
from the partition name by an underscore. For example, "logs_42"
would be the container name for partition 42.
+ Note that you will want to set this to a value that uniquely
+ identifies your Zuul installation if using shared object stores that
+ require globally unique container names. For example if using a
+ public cloud whose Swift API is provided by Ceph.
+
.. zuul:rolevar:: zuul_log_container_public
:default: true
@@ -60,3 +65,14 @@ This uploads logs to an OpenStack Object Store (Swift) container.
Whether to create `index.html` files with directory indexes. If set
to false, Swift containers can be marked with a `Web-Listings=true`
property to activate Swift's own directory indexing.
+
+.. zuul:rolevar:: zuul_log_path_shard_build
+ :default: False
+
+ This var is consumed by set-zuul-log-path-fact which upload-logs-swift
+ calls into. If you set this you will get log paths prefixed with the
+ first three characters of the build uuid. This will improve log file
+ sharding.
+
+ More details can be found at
+ :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`.
diff --git a/roles/upload-logs-swift/library/test_zuul_swift_upload.py b/roles/upload-logs-swift/library/test_zuul_swift_upload.py
index 39f23b4..075fa9e 100644
--- a/roles/upload-logs-swift/library/test_zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/test_zuul_swift_upload.py
@@ -20,10 +20,12 @@ __metaclass__ = type
import os
import testtools
+import time
+import stat
import fixtures
from bs4 import BeautifulSoup
-from .zuul_swift_upload import FileList, Indexer
+from .zuul_swift_upload import FileList, Indexer, FileDetail
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
@@ -357,3 +359,24 @@ class TestFileList(testtools.TestCase):
self.assertEqual(rows[0].find('a').get('href'), 'subdir.txt')
self.assertEqual(rows[0].find('a').text, 'subdir.txt')
+
+
+class TestFileDetail(testtools.TestCase):
+
+ def test_get_file_detail(self):
+ '''Test files info'''
+ path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
+ file_detail = FileDetail(path, '')
+ path_stat = os.stat(path)
+ self.assertEqual(
+ time.gmtime(path_stat[stat.ST_MTIME]),
+ file_detail.last_modified)
+ self.assertEqual(16, file_detail.size)
+
+ def test_get_file_detail_missing_file(self):
+ '''Test files that go missing during a walk'''
+
+ file_detail = FileDetail('missing/file/that/we/cant/find', '')
+
+ self.assertEqual(time.gmtime(0), file_detail.last_modified)
+ self.assertEqual(0, file_detail.size)
diff --git a/roles/upload-logs-swift/library/zuul_swift_upload.py b/roles/upload-logs-swift/library/zuul_swift_upload.py
index de53393..196bc75 100755
--- a/roles/upload-logs-swift/library/zuul_swift_upload.py
+++ b/roles/upload-logs-swift/library/zuul_swift_upload.py
@@ -38,6 +38,7 @@ import sys
import tempfile
import threading
import time
+import traceback
import zlib
import collections
@@ -45,7 +46,7 @@ import openstack
import requests
import requests.exceptions
import requestsexceptions
-import keystoneauth1
+import keystoneauth1.exceptions
from ansible.module_utils.basic import AnsibleModule
@@ -133,6 +134,26 @@ def get_mime_icon(mime, filename=''):
return "data:image/png;base64,%s" % ICON_IMAGES[icon]
+def get_cloud(cloud):
+ if isinstance(cloud, dict):
+ config = openstack.config.loader.OpenStackConfig().get_one(**cloud)
+ return openstack.connection.Connection(config=config)
+ else:
+ return openstack.connect(cloud=cloud)
+
+
+def retry_function(func):
+ for attempt in range(1, POST_ATTEMPTS + 1):
+ try:
+ return func()
+ except Exception:
+ if attempt >= POST_ATTEMPTS:
+ raise
+ else:
+ logging.exception("Error on attempt %d" % attempt)
+ time.sleep(attempt * 10)
+
+
def sizeof_fmt(num, suffix='B'):
# From http://stackoverflow.com/questions/1094841/
# reusable-library-to-get-human-readable-version-of-file-size
@@ -157,6 +178,12 @@ class FileDetail():
used for links.
filename (str): An optional alternate filename in links.
"""
+ # Make FileNotFoundError exception to be compatible with python2
+ try:
+ FileNotFoundError # noqa: F823
+ except NameError:
+ FileNotFoundError = OSError
+
self.full_path = full_path
if filename is None:
self.filename = os.path.basename(full_path)
@@ -173,11 +200,11 @@ class FileDetail():
self.mimetype = 'application/directory'
self.encoding = None
self.folder = True
- if self.full_path:
+ try:
st = os.stat(self.full_path)
self.last_modified = time.gmtime(st[stat.ST_MTIME])
self.size = st[stat.ST_SIZE]
- else:
+ except (FileNotFoundError, TypeError):
self.last_modified = time.gmtime(0)
self.size = 0
@@ -462,11 +489,7 @@ class DeflateFilter():
class Uploader():
def __init__(self, cloud, container, prefix=None, delete_after=None,
public=True):
- if isinstance(cloud, dict):
- config = openstack.config.loader.OpenStackConfig().get_one(**cloud)
- self.cloud = openstack.connection.Connection(config=config)
- else:
- self.cloud = openstack.connect(cloud=cloud)
+ self.cloud = cloud
self.container = container
self.prefix = prefix or ''
self.delete_after = delete_after
@@ -487,30 +510,36 @@ class Uploader():
cdn_url = None
if not self.cloud.get_container(self.container):
- self.cloud.create_container(name=self.container, public=public)
- self.cloud.update_container(
- name=self.container,
- headers={'X-Container-Meta-Web-Index': 'index.html',
- 'X-Container-Meta-Access-Control-Allow-Origin': '*'})
+ retry_function(
+ lambda: self.cloud.create_container(
+ name=self.container, public=public))
+ headers = {'X-Container-Meta-Web-Index': 'index.html',
+ 'X-Container-Meta-Access-Control-Allow-Origin': '*'}
+ retry_function(
+ lambda: self.cloud.update_container(
+ name=self.container,
+ headers=headers))
# 'X-Container-Meta-Web-Listings': 'true'
# The ceph radosgw swift implementation requires an
# index.html at the root in order for any other indexes to
# work.
index_headers = {'access-control-allow-origin': '*'}
- self.cloud.create_object(self.container,
- name='index.html',
- data='',
- content_type='text/html',
- **index_headers)
+ retry_function(
+ lambda: self.cloud.create_object(self.container,
+ name='index.html',
+ data='',
+ content_type='text/html',
+ **index_headers))
# Enable the CDN in rax
if cdn_url:
- self.cloud.session.put(cdn_url)
+ retry_function(lambda: self.cloud.session.put(cdn_url))
if cdn_url:
- endpoint = (self.cloud.session.head(cdn_url)
- .headers['X-Cdn-Ssl-Uri'])
+ endpoint = retry_function(
+ lambda: self.cloud.session.head(
+ cdn_url).headers['X-Cdn-Ssl-Uri'])
container = endpoint
else:
endpoint = self.cloud.object_store.get_endpoint()
@@ -542,7 +571,7 @@ class Uploader():
logging.debug("%s: processing job %s",
threading.current_thread(),
file_detail)
- self._post_file(file_detail)
+ retry_function(lambda: self._post_file(file_detail))
except requests.exceptions.RequestException:
# Do our best to attempt to upload all the files
logging.exception("Error posting file after multiple attempts")
@@ -580,32 +609,24 @@ class Uploader():
# This is required for Rackspace CDN
headers['access-control-allow-origin'] = '*'
- for attempt in range(1, POST_ATTEMPTS + 1):
- try:
- if not file_detail.folder:
- if (file_detail.encoding is None and
- self._is_text_type(file_detail.mimetype)):
- headers['content-encoding'] = 'deflate'
- data = DeflateFilter(open(file_detail.full_path, 'rb'))
- else:
- if file_detail.encoding:
- headers['content-encoding'] = file_detail.encoding
- data = open(file_detail.full_path, 'rb')
- else:
- data = ''
- relative_path = relative_path.rstrip('/')
- if relative_path == '':
- relative_path = '/'
- self.cloud.create_object(self.container,
- name=relative_path,
- data=data,
- **headers)
- break
- except requests.exceptions.RequestException:
- logging.exception(
- "File posting error on attempt %d" % attempt)
- if attempt >= POST_ATTEMPTS:
- raise
+ if not file_detail.folder:
+ if (file_detail.encoding is None and
+ self._is_text_type(file_detail.mimetype)):
+ headers['content-encoding'] = 'deflate'
+ data = DeflateFilter(open(file_detail.full_path, 'rb'))
+ else:
+ if file_detail.encoding:
+ headers['content-encoding'] = file_detail.encoding
+ data = open(file_detail.full_path, 'rb')
+ else:
+ data = ''
+ relative_path = relative_path.rstrip('/')
+ if relative_path == '':
+ relative_path = '/'
+ self.cloud.create_object(self.container,
+ name=relative_path,
+ data=data,
+ **headers)
def run(cloud, container, files,
@@ -669,16 +690,27 @@ def ansible_main():
)
p = module.params
- url = run(p.get('cloud'), p.get('container'), p.get('files'),
- indexes=p.get('indexes'),
- parent_links=p.get('parent_links'),
- topdir_parent_link=p.get('topdir_parent_link'),
- partition=p.get('partition'),
- footer=p.get('footer'),
- delete_after=p.get('delete_after', 15552000),
- prefix=p.get('prefix'),
- public=p.get('public'))
-
+ cloud = get_cloud(p.get('cloud'))
+ try:
+ url = run(cloud, p.get('container'), p.get('files'),
+ indexes=p.get('indexes'),
+ parent_links=p.get('parent_links'),
+ topdir_parent_link=p.get('topdir_parent_link'),
+ partition=p.get('partition'),
+ footer=p.get('footer'),
+ delete_after=p.get('delete_after', 15552000),
+ prefix=p.get('prefix'),
+ public=p.get('public'))
+ except (keystoneauth1.exceptions.http.HttpError,
+ requests.exceptions.RequestException):
+ s = "Error uploading to %s.%s" % (cloud.name, cloud.config.region_name)
+ logging.exception(s)
+ s += "\n" + traceback.format_exc()
+ module.fail_json(
+ changed=False,
+ msg=s,
+ cloud=cloud.name,
+ region_name=cloud.config.region_name)
module.exit_json(changed=True,
url=url)
@@ -738,7 +770,7 @@ def cli_main():
if append_footer.lower() == 'none':
append_footer = None
- url = run(args.cloud, args.container, args.files,
+ url = run(get_cloud(args.cloud), args.container, args.files,
indexes=not args.no_indexes,
parent_links=not args.no_parent_links,
topdir_parent_link=args.create_topdir_parent_link,
diff --git a/roles/upload-logs/README.rst b/roles/upload-logs/README.rst
index 6669f3d..7df56e3 100644
--- a/roles/upload-logs/README.rst
+++ b/roles/upload-logs/README.rst
@@ -45,3 +45,14 @@ description of the site_logs secret in this example post-run playbook:
when the job has failed.
.. note:: Intended to be set by admins via site-variables.
+
+.. zuul:rolevar:: zuul_log_path_shard_build
+ :default: False
+
+ This var is consumed by set-zuul-log-path-fact which upload-logs
+ calls into. If you set this you will get log paths prefixed with the
+ first three characters of the build uuid. This will improve log file
+ sharding.
+
+ More details can be found at
+ :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`.
diff --git a/zuul.d/js-jobs.yaml b/zuul.d/js-jobs.yaml
index 48b4b6d..e12baa9 100644
--- a/zuul.d/js-jobs.yaml
+++ b/zuul.d/js-jobs.yaml
@@ -191,6 +191,6 @@
Path to operate in.
post-run: playbooks/tox/docs-post.yaml
- success-url: html/
+ success-url: docs/
vars:
npm_command: docs
diff --git a/zuul.d/python-jobs.yaml b/zuul.d/python-jobs.yaml
index c3fd68d..56aec9a 100644
--- a/zuul.d/python-jobs.yaml
+++ b/zuul.d/python-jobs.yaml
@@ -115,7 +115,7 @@
post-run:
- playbooks/tox/docs-post.yaml
- playbooks/tox/post.yaml
- success-url: html/
+ success-url: docs/
- job:
name: tox-linters
@@ -307,7 +307,7 @@
:default: {{ zuul.project.src_dir }}
Directory to operate in.
- success-url: html/
+ success-url: docs/
# Release notes always build on master.
override-checkout: master
pre-run: playbooks/releasenotes/pre.yaml