summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.zuul.yaml28
-rw-r--r--LICENSE202
-rw-r--r--README.rst40
-rw-r--r--doc/conf.py11
l---------doc/index.rst1
-rw-r--r--npmfed/__init__.py0
-rwxr-xr-xnpmfed/cmd.py211
-rw-r--r--npmfed/packagetospec.py79
-rw-r--r--npmfed/parsers.py168
-rw-r--r--npmfed/tests/__init__.py0
-rw-r--r--npmfed/tests/test_units.py19
-rw-r--r--npmfed/yarntospec.py340
-rw-r--r--requirements.txt1
-rw-r--r--setup.cfg41
-rwxr-xr-xsetup.py5
-rw-r--r--test-requirements.txt1
-rw-r--r--tox.ini18
17 files changed, 1165 insertions, 0 deletions
diff --git a/.zuul.yaml b/.zuul.yaml
new file mode 100644
index 0000000..a16d22b
--- /dev/null
+++ b/.zuul.yaml
@@ -0,0 +1,28 @@
+- project:
+ check:
+ jobs:
+ - tox-pep8:
+ nodeset:
+ nodes:
+ - name: testrunner
+ label: runc-fedora
+ - tox-py35:
+ nodeset:
+ nodes:
+ - name: testrunner
+ label: runc-fedora
+ gate:
+ jobs:
+ - tox-pep8:
+ nodeset:
+ nodes:
+ - name: testrunner
+ label: runc-fedora
+ - tox-py35:
+ nodeset:
+ nodes:
+ - name: testrunner
+ label: runc-fedora
+ release:
+ jobs:
+ - upload-pypi
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..10d9044
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,202 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..2367f3f
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,40 @@
+npmfed
+======
+
+NPM packaging tool.
+
+Usage
+-----
+
+Prepare yarn.lock:
+
+.. code-block:: bash
+
+ rm -rf yarn.lock node_modules
+ yes 1 | yarn install --flat
+
+Make sure package is functional with flat dependencies, if not:
+
+- install upstream node_modules and keep a copy to compare version
+- use 'resolution' list in package.json to adjust selected flat versions
+- watch out for nested dependencies,
+ e.g. loader-utils for html-webpack-plugin needs to be 0.2.17
+
+Build packages:
+
+.. code-block:: bash
+
+ npmfed spec git/grafana/grafana
+
+
+TODO
+----
+
+- Implement repo metadata and remove hardcoded grafana information in Repo obj
+
+
+Contribute
+----------
+
+Contribution are most welcome, use **git-review** to propose a change.
+Setup your ssh keys after sign in https://softwarefactory-project.io/auth/login
diff --git a/doc/conf.py b/doc/conf.py
new file mode 100644
index 0000000..a639596
--- /dev/null
+++ b/doc/conf.py
@@ -0,0 +1,11 @@
+import sphinx_rtd_theme
+extensions = []
+source_suffix = '.rst'
+master_doc = 'index'
+project = 'npmfed'
+copyright = '2018, Red Hat'
+author = 'Tristan Cacqueray'
+pygments_style = 'sphinx'
+todo_include_todos = False
+html_theme = 'sphinx_rtd_theme'
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
diff --git a/doc/index.rst b/doc/index.rst
new file mode 120000
index 0000000..89a0106
--- /dev/null
+++ b/doc/index.rst
@@ -0,0 +1 @@
+../README.rst \ No newline at end of file
diff --git a/npmfed/__init__.py b/npmfed/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/npmfed/__init__.py
diff --git a/npmfed/cmd.py b/npmfed/cmd.py
new file mode 100755
index 0000000..b806a8b
--- /dev/null
+++ b/npmfed/cmd.py
@@ -0,0 +1,211 @@
+#!/bin/env python3
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# The command line interface
+
+import click
+import copy
+import glob
+import json
+import os
+import urllib.request
+from urllib.error import HTTPError
+
+from npmfed.parsers import Repo
+from npmfed.packagetospec import PackageToSpec
+from npmfed.yarntospec import build
+
+
+GEN_PKG_DB = """repodata='Set this to current primary list url'
+curl ${repodata} | grep '"npm(.*)"' | sed -e 's/^.*npm(//' -e 's/).*//' | \
+ sort -u > %s
+"""
+
+
+@click.group()
+def cli():
+ pass
+
+
+@cli.command(help="Graph a repository dependencies")
+@click.option('--dev/--no-dev', default=False)
+@click.argument('repodir', type=click.Path(exists=True, file_okay=False))
+def graph(dev, repodir):
+ repo = Repo(repodir, dev)
+
+ # Generate dot graph
+ dot = [
+ "// %s dependencies count: %d" % (repo.package.name, len(repo.graph)),
+ "digraph node_dep {"
+ ]
+ # colorize leaf
+ for name, dependencies in repo.graph.items():
+ if not dependencies:
+ dot.append(' "%s" [style = filled,color=antiquewhite];' % name)
+ for name, dependencies in repo.graph.items():
+ for dependency in dependencies:
+ dot.append(' "%s" -> "%s";' % (name, dependency))
+ dot.append("}")
+ print("\n".join(dot))
+
+
+@cli.command(help="Repo to spec")
+@click.option("--rpmdevtree", default=os.path.expanduser("~/rpmbuild"),
+ type=click.Path(exists=True, file_okay=False))
+@click.option("--yarncache", default=os.path.expanduser("~/.cache/yarn/v1"),
+ type=click.Path(exists=True, file_okay=False))
+@click.option('--dev/--no-dev', default=True)
+@click.option('--bail/--no-bail', default=True)
+@click.option('--skip')
+@click.argument('repodir', type=click.Path(exists=True, file_okay=False))
+def spec(rpmdevtree, yarncache, dev, bail, skip, repodir):
+ repo = Repo(repodir, dev)
+ print("(1/4) Checking if the package already exists in PkgDB")
+ # TODO: check if repo.package.name already exists
+
+ print("(2/4) Checking if dependencies are missing in PkgDB")
+ local_cache = ".cache-npm-provided-by-rawhide"
+ if os.path.exists(local_cache):
+ existing = set(map(lambda x: x[:-1], open(local_cache).readlines()))
+ # TODO: check for correct versions...
+ else:
+ print("Couldn't find local PkgDB cache, please run:")
+ print(GEN_PKG_DB % local_cache)
+ exit()
+
+ print("(3/4) Build missing dependencies")
+ build_order = repo.build_order(existing)
+ idx = 0
+ if skip:
+ for dependency in build_order:
+ if dependency == skip:
+ break
+ idx += 1
+ print("Building %d packages" % (len(build_order) - idx))
+ for dependency in build_order[idx:]:
+ if dependency == repo.package.name:
+ continue
+ package = repo.yarn.packages[dependency]
+ try:
+ base = yarncache
+ if "/" in package["name"]:
+ base += "/npm-%s" % package["name"].split('/')[0]
+ metadata = getMetadata(package["hash"], base)
+ except RuntimeError as e:
+ print(e)
+ continue
+ try:
+ build(rpmdevtree, metadata)
+ except Exception:
+ if bail:
+ raise
+ print("Failed to build %s" % metadata["manifest"]["name"])
+
+ print("(4/4) Build package")
+ final_spec = os.path.join(rpmdevtree, "SPECS", "%s.spec" % repo.name)
+ print(final_spec)
+ if not os.path.exists(final_spec):
+ with open(final_spec, "w") as fileobj:
+ fileobj.write(PackageToSpec(repo).render())
+
+
+@cli.command(help="Npm to spec")
+@click.option("--rpmdevtree", default=os.path.expanduser("~/rpmbuild"),
+ type=click.Path(exists=True, file_okay=False))
+@click.option('--dev/--no-dev', default=True)
+@click.argument("name")
+@click.argument("version", required=False)
+def npm(rpmdevtree, dev, name, version=None):
+ yarn_registry_cache = os.path.join(rpmdevtree, "YARN_CACHE")
+ os.makedirs(yarn_registry_cache, exist_ok=True)
+
+ def get_yarn(name, version):
+ cache_data = os.path.join(
+ yarn_registry_cache, "%s.json" % name.replace('/', '_'))
+
+ if not os.path.exists(cache_data):
+ r = urllib.request.urlopen("https://registry.yarnpkg.com/" + name)
+ with open(cache_data, "wb") as of:
+ of.write(r.read())
+ data = json.load(open(cache_data))
+ # Always use latest...
+ version = "latest"
+ if version is None or version == "latest":
+ version = data["dist-tags"]["latest"]
+ manifest = data["versions"].get(version)
+ if not manifest:
+ raise RuntimeError("%s: %s doesn't exists" % (name, version))
+ return manifest
+
+ print("(1/2) Grab metadata from yarnpk registry")
+ packages = {}
+ graph = {}
+ packages[name] = get_yarn(name, version)
+ deps = [name]
+ while deps:
+ dep = deps.pop()
+ dep_req = copy.copy(packages[dep].get("dependencies", {}))
+ if dev:
+ dep_req.update(copy.copy(packages[dep].get("devDependencies", {})))
+ graph[dep] = []
+ for dep_name, dep_version in dep_req.items():
+ graph[dep].append(dep_name)
+ print("%s needs %s" % (dep, dep_name))
+ if dep_name in packages:
+ if packages[dep_name]["version"] != dep_version:
+ print("%s: multiple version %s and %s" % (
+ dep_name, packages[dep_name]["version"], dep_version))
+ continue
+ try:
+ packages[dep_name] = get_yarn(dep_name, dep_version)
+ except (HTTPError, json.decoder.JSONDecodeError):
+ print("Can't find %s" % dep_name)
+ cache_data = os.path.join(
+ yarn_registry_cache, "%s.json" % name.replace('/', '_'))
+ open(cache_data, "w").close()
+ deps.append(dep_name)
+
+ print("(2/2) Build packages")
+ needed = set(packages.keys())
+ needed.remove(name)
+ existing = set()
+ needed_dependency = sorted(list(needed - existing))
+ build_order = []
+
+ def walk(package):
+ if package in build_order or package in existing:
+ return
+ dependencies = graph[package]
+ while len(dependencies):
+ walk(dependencies.pop())
+ build_order.append(package)
+ while needed_dependency:
+ walk(needed_dependency.pop())
+ build_order.append(name)
+
+ for dependency in build_order:
+ package = packages[dependency]
+ build(rpmdevtree, {"manifest": package})
+
+
+def getMetadata(yarn_hash, base):
+ # Quick and dirty yarn metadata lookup...
+ metadata_file = glob.glob("%s/*%s*/.yarn-metadata.json" % (
+ base, yarn_hash))
+ if not metadata_file:
+ raise RuntimeError("%s: couldn't find in %s" % (yarn_hash, base))
+ if len(metadata_file) != 1:
+ raise RuntimeError("%s: multiple hit in %s" % (yarn_hash, base))
+ return json.load(open(metadata_file[0]))
diff --git a/npmfed/packagetospec.py b/npmfed/packagetospec.py
new file mode 100644
index 0000000..e3cbf82
--- /dev/null
+++ b/npmfed/packagetospec.py
@@ -0,0 +1,79 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# The final package spec generator
+
+SPEC = """%{{?nodejs_find_provides_and_requires}}
+
+Name: {name}
+Version: {version}
+Release: 1%{{?dist}}
+Summary: {summary}
+
+License: {license}
+URL: {homepage}
+Source0: {source}
+
+ExclusiveArch: %{{nodejs_arches}} noarch
+BuildArch: noarch
+
+BuildRequires: nodejs-packaging
+{buildRequires}
+
+%description
+{summary}
+
+
+%prep
+%autosetup -n %{name}-%{version}
+rm -Rf node_modules
+
+%build
+react-scripts build
+
+%install
+mkdir -p %{{buildroot}}/%{{_datadir}}/{name}
+mv build/* %{{buildroot}}/%{{_datadir}}/{name}/
+
+%nodejs_symlink_deps
+
+%files
+%doc
+%license
+%{{buildroot}}/%{{_datadir}}/{name}
+
+%changelog
+"""
+
+
+class PackageToSpec:
+ def __init__(self, repo):
+ self.repo = repo
+
+ def render(self):
+ # Generate build requires list
+ buildRequires = []
+ d = self.repo.package.dependencies + self.repo.package.devDependencies
+ for dependency in d:
+ buildRequires.append("BuildRequires: npm(%s)" % dependency)
+
+ return SPEC.format(
+ name=self.repo.name,
+ version=self.repo.version,
+ summary=self.repo.description,
+ license=self.repo.license,
+ source=self.repo.src,
+ homepage=self.repo.homepage,
+ buildRequires="\n".join(buildRequires)
+ )
diff --git a/npmfed/parsers.py b/npmfed/parsers.py
new file mode 100644
index 0000000..0540746
--- /dev/null
+++ b/npmfed/parsers.py
@@ -0,0 +1,168 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Parsers for package.json and yarn.log
+
+import json
+import os.path
+import copy
+
+
+DEDUP_COMMAND = """
+rm -rf yarn.lock node_modules && \
+ yes 1 | yarn install --flat --production --ignore-scripts && \
+ git checkout package.json && yes 1 | yarn install --flat
+"""
+
+
+class PackageFile:
+ """Parse a package.json and extract relevant informations"""
+ def __init__(self, fileobj):
+ package = json.load(fileobj)
+ self.dependencies = []
+ self.devDependencies = []
+ self.name = package["name"]
+ self.license = package.get('license', 'AS-IS')
+ self.version = package["version"]
+ if not self.license:
+ raise RuntimeError("%s: missing license" % fileobj.name)
+ for dependency in package.get("dependencies", {}).keys():
+ self.dependencies.append(dependency)
+ for dependency in package.get("devDependencies", {}).keys():
+ self.devDependencies.append(dependency)
+
+
+class YarnFile:
+ """Parse a yarn.lock file and extract dependency list"""
+ def __init__(self, fileobj):
+ self.packages = {}
+ self.errors = False
+ # Check headers
+ fileobj.readline()
+ if fileobj.readline() != "# yarn lockfile v1\n":
+ raise RuntimeError(
+ "%s: missing 'yarn lockfile v1' header" % fileobj.name)
+ # Read package blocks
+ package = {}
+ while True:
+ line = fileobj.readline()
+ if not line:
+ break
+ if line == '\n':
+ continue
+ if line[0] != ' ':
+ # New package definition begin with no ' ' prefix
+ if package:
+ # Save already parsed package
+ self.add_package(package)
+ # Strip name and only keep the first component ',' separated
+ strip = line.replace('"', '').split(',')[0]
+ package = {
+ 'name': strip[:strip.rindex('@')],
+ 'dependencies': [],
+ 'hash': None,
+ }
+ # Add package metadata
+ if line.startswith(' version'):
+ package['version'] = line.split()[1][1:-1]
+ elif line.startswith(' resolved'):
+ if '#' in line:
+ hash_sep = '#'
+ else:
+ hash_sep = '/'
+ package['hash'] = line.split(hash_sep)[-1][:-2]
+ elif line.startswith(' '):
+ # Dependencies line begin with 4 space
+ package['dependencies'].append(
+ # Only keep the first component, discard version numbers
+ line.replace('"', '').split(None, 1)[0])
+
+ # Save last package
+ self.add_package(package)
+
+ if self.errors:
+ raise RuntimeError("Duplicated requirements detected, "
+ "fix the yarn.lock file using:\n" +
+ DEDUP_COMMAND)
+
+ def add_package(self, package):
+ if package['name'] in self.packages:
+ self.errors = True
+ print("Duplicate package %s and %s" %
+ (package, self.packages[package["name"]]))
+ self.packages[package['name']] = package
+
+
+class Repo:
+ """Parse a repository informations and generate a dependency graph"""
+ def __init__(self, path, include_dev):
+ if not path:
+ self.name = "unit-test"
+ return
+ self.name = os.path.basename(path.rstrip('/'))
+ package_file = os.path.join(path, "package.json")
+ if not os.path.exists(package_file):
+ raise RuntimeError("%s: does not exist" % package_file)
+
+ yarn_file = os.path.join(path, "yarn.lock")
+ if not os.path.exists(yarn_file):
+ raise RuntimeError("%s: does not exist" % yarn_file)
+
+ with open(package_file) as package_obj, open(yarn_file) as yarn_obj:
+ self.process(package_obj, yarn_obj, include_dev)
+
+ def process(self, package_obj, yarn_obj, include_dev):
+ self.package = PackageFile(package_obj)
+ self.yarn = YarnFile(yarn_obj)
+ self.graph = {}
+
+ # TODO: get from git
+ self.version = "0.0.0"
+ self.homepage = "https://github.com/u/r"
+ self.src = "https://github.com/u/r/archive/%{version}.tar.gz"
+ self.description = "A packaged application"
+ self.license = "ASL-2.0"
+
+ # Load dependencies from package.json
+ dependencies = self.package.dependencies
+ if include_dev:
+ dependencies += self.package.devDependencies
+ self.graph[self.package.name] = dependencies
+
+ # Load dependencies' dependencies from yarn.lock
+ to_load = copy.copy(dependencies)
+ while len(to_load):
+ dependency = to_load.pop()
+ if dependency in self.graph:
+ continue
+ dependencies = self.yarn.packages[dependency]['dependencies']
+ self.graph[dependency] = sorted(dependencies)
+ to_load.extend(dependencies)
+
+ def build_order(self, existing=set()):
+ needed = set(self.graph.keys())
+ needed.remove(self.package.name)
+ needed_dependency = sorted(list(needed - existing))
+ build_order = []
+
+ def walk(package):
+ if package in build_order or package in existing:
+ return
+ dependencies = self.graph[package]
+ while len(dependencies):
+ walk(dependencies.pop())
+ build_order.append(package)
+ while needed_dependency:
+ walk(needed_dependency.pop())
+ return build_order
diff --git a/npmfed/tests/__init__.py b/npmfed/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/npmfed/tests/__init__.py
diff --git a/npmfed/tests/test_units.py b/npmfed/tests/test_units.py
new file mode 100644
index 0000000..716905b
--- /dev/null
+++ b/npmfed/tests/test_units.py
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import pytest
+
+
+class TestLockFile(object):
+ def test_basic(self):
+ with pytest.raises(RuntimeError):
+ raise RuntimeError("todo...")
diff --git a/npmfed/yarntospec.py b/npmfed/yarntospec.py
new file mode 100644
index 0000000..84bd1f3
--- /dev/null
+++ b/npmfed/yarntospec.py
@@ -0,0 +1,340 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os.path
+
+SPEC = """%{{?nodejs_find_provides_and_requires}}
+%global packagename {packagename}
+
+Name: nodejs-{name}
+Version: {version}
+Release: 1%{{?dist}}
+Summary: {summary}
+
+License: {license}
+URL: {homepage}
+Source0: {source}
+
+ExclusiveArch: %{{nodejs_arches}} noarch
+BuildArch: noarch
+
+BuildRequires: nodejs-packaging
+{buildRequires}
+
+Provides: npm(%{{packagename}})
+
+%description
+{summary}
+
+%prep
+%setup -n "{src_dir_name}"
+rm -Rf node_modules
+{fix_dep}
+
+%build
+# nothing to do!
+
+%install
+mkdir -p %{{buildroot}}%{{nodejs_sitelib}}/%{{packagename}}
+cp -pr * \
+ %{{buildroot}}%{{nodejs_sitelib}}/%{{packagename}}
+
+%{{nodejs_symlink_deps}}{subname}
+"""
+
+SPEC_CHECK = """
+%check
+%{{nodejs_symlink_deps}} --check
+%{{__nodejs}} -e 'require("./")'
+"""
+
+SPEC_FOOTER = """
+%files
+%doc
+%license
+%{{nodejs_sitelib}}/%{{packagename}}
+
+%changelog
+"""
+
+# Some package doesn't use the default 'package' directory name...
+PACKAGE_SRC_DIR_NAME = {
+ "remarkable-1.7.1": "jonschlinkert-remarkable-ae85f6c",
+ "ordered-esprima-props-1.1.0": "oep",
+ "ordered-ast-traverse-1.1.1": "oat",
+ "ng-annotate-1.2.1": "ng-annotate",
+ "ejs-2.6.1": "ejs-v2.6.1",
+ "types-node-10.9.4": "node",
+ "@types/webpack": "webpack v3",
+}
+
+PACKAGE_SKIP_TEST = (
+ "webpack-core",
+ "fsevents",
+ # Error: Cannot find module 'webpack/lib/RequestShortener'
+ "uglifyjs-webpack-plugin",
+ # Error: Cannot find module 'events/'
+ "node-libs-browser",
+ # ReferenceError: Cache is not defined
+ "serviceworker-cache-polyfill",
+ # ReferenceError: self is not defined
+ "sw-toolbox",
+ # Error: Cannot find module 'uglify-js'
+ "sw-precache-webpack-plugin",
+ # Error: Cannot find module 'nth-check'
+ "renderkid",
+ # Error: Cannot find module 'lodash/isPlainObject'
+ "babel-types",
+ # ReferenceError: window is not defined
+ "react-error-overlay",
+ # Error: Cannot find module 'react'
+ "react-dom",
+ # Error: Cannot find module './'
+ "react-dev-utils",
+ # Error: Cannot find module 'nth-check'
+ "pretty-error",
+ # Error: Cannot find module 'lodash/isPlainObject'
+ "babel-traverse",
+ # Error: Cannot find module 'lodash/cloneDeep'
+ "babel-template",
+)
+
+
+def npm2rpm(npm):
+ """Convert npm name to rpm"""
+ return npm.replace('@', '').replace('/', '-').replace('.', '-dot-')
+
+
+class YarnToSpec:
+ def __init__(self, yarn_metadata):
+ self.yarn_metadata = yarn_metadata
+ self.native = False
+ self.fixdep = []
+ self.add_check = True
+
+ def render(self):
+ manifest = self.yarn_metadata["manifest"]
+
+ if manifest["name"] in PACKAGE_SKIP_TEST:
+ self.add_check = False
+
+ subname = ""
+ if "/" in manifest["name"]:
+ subname = "/" + os.path.dirname(manifest["name"])
+
+ # %autosetup directory name
+ src_dir_name = PACKAGE_SRC_DIR_NAME.get("%s-%s" % (
+ manifest["name"], manifest["version"]), "package")
+ if src_dir_name == "package":
+ # Try without version too
+ src_dir_name = PACKAGE_SRC_DIR_NAME.get(
+ manifest["name"], "package")
+
+ # Generate build requires list
+ buildRequires = []
+ for dependency in manifest.get("dependencies", []):
+ buildRequires.append(
+ "BuildRequires: npm(%s)" % dependency)
+
+ fix_dep = []
+ for fixdep in self.fixdep:
+ fix_dep.append("%nodejs_fixdep " + fixdep)
+
+ # Discover homepage
+ homepage = manifest.get("homepage")
+ if not homepage:
+ homepage = manifest.get("repository", {}).get("url")
+ if not homepage:
+ homepage = "https://www.npmjs.com/package/%s" % (
+ manifest["name"])
+ if "/tree/master/" in homepage:
+ homepage = homepage[:homepage.rindex("/tree/master/")]
+
+ # Check license
+ if not manifest.get("license"):
+ print("%s: unknown license" % manifest["name"])
+
+ # Source url
+ source = (
+ "http://registry.npmjs.org/"
+ "%{packagename}/-/%{packagename}-%{version}.tgz"
+ )
+
+ # summary
+ summary = manifest.get("description")
+ if not summary:
+ summary = 'Nodejs package for %s' % manifest['name']
+
+ # Disable symlink deps when it doesn't work for sub dir packages
+ symlink_deps = "1"
+ deps = ''.join(list(manifest.get('dependencies', [])) +
+ list(manifest.get('devDependencies', [])) +
+ list(manifest.get('optionalDependencies', [])))
+ if '/' in manifest['name'] or '/' in deps:
+ symlink_deps = "0"
+ self.add_check = False
+
+ format_args = dict(
+ name=npm2rpm(manifest["name"]),
+ subname=subname,
+ packagename=manifest["name"],
+ version=manifest["version"],
+ src_dir_name=src_dir_name,
+ summary=summary,
+ license=manifest.get("license", 'AS-IS'),
+ source=source,
+ homepage=homepage,
+ buildRequires="\n".join(buildRequires),
+ fix_dep="\n".join(fix_dep),
+ fix_symlink_deps=symlink_deps,
+ )
+ spec_content = SPEC.format(**format_args)
+ if self.add_check:
+ spec_content += SPEC_CHECK.format(**format_args)
+ spec_content += SPEC_FOOTER.format(**format_args)
+ return spec_content
+
+
+def build(rpmdevtree, metadata):
+ import os
+ import subprocess
+ import shutil
+
+ def execute(*argv, fatal=True):
+ print(argv)
+ p = subprocess.Popen(
+ argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ rc = p.wait()
+ if rc and fatal:
+ raise RuntimeError("Failed to run: %s" % " ".join(argv))
+ return rc, stdout.decode('utf-8'), stderr.decode('utf-8')
+
+ def get_nvr(spec_file):
+ spec = execute("rpmspec", "--define", "dist .fc30", "-P", spec_file)[1]
+ nvr = {}
+ for line in spec.split('\n'):
+ for k in ('Name', 'Version', 'Release'):
+ if line.startswith('%s:' % k):
+ nvr[k] = line.split()[-1]
+ return "{Name}-{Version}-{Release}".format(**nvr), nvr["Version"]
+
+ # Check if already built
+ spec_file = os.path.join(
+ rpmdevtree, "SPECS",
+ "nodejs-%s.spec" % npm2rpm(metadata["manifest"]["name"]))
+
+ try:
+ spec = YarnToSpec(metadata)
+ if metadata["manifest"]["name"] in ("node-sass", ):
+ spec.native = True
+ if not os.path.exists(spec_file):
+ spec_content = spec.render()
+ with open(spec_file, "w") as fileobj:
+ fileobj.write(spec_content)
+ except Exception:
+ import pprint
+ print("Failed to format spec for")
+ pprint.pprint(metadata)
+ raise
+
+ nvr, v = get_nvr(spec_file)
+
+ if v != metadata["manifest"]["version"]:
+ # Update spec to bump version
+ print("Version mis-match %s != %s" % (
+ v, metadata["manifest"]["version"]))
+ #spec_content = spec.render()
+ #with open(spec_file, "w") as fileobj:
+ # fileobj.write(spec_content)
+ #nvr, v = get_nvr(spec_file)
+
+ if not spec.native:
+ rpm_file = os.path.join(
+ rpmdevtree, "RPMS", "noarch", "%s.noarch.rpm" % nvr)
+ else:
+ rpm_file = os.path.join(
+ rpmdevtree, "RPMS", "x86_64", "%s.x86_64.rpm" % nvr)
+ if os.path.exists(rpm_file):
+ print("%s: already built %s" % (spec_file, rpm_file))
+ return
+ print("\n\033[93m%s: building...\033[0m" % spec_file)
+ execute(
+ "spectool", "-C", os.path.join(rpmdevtree, "SOURCES"), "-g", spec_file)
+ execute("rpmbuild", "-bs", spec_file)
+ srpm_file = os.path.join(rpmdevtree, "SRPMS", "%s.src.rpm" % nvr)
+ if not os.path.exists(srpm_file):
+ raise RuntimeError("%s doestn't exist" % srpm_file)
+ execute("sudo", "dnf", "builddep", "-y", srpm_file)
+
+ shutil.rmtree(os.path.join(rpmdevtree, "BUILD"))
+ os.mkdir(os.path.join(rpmdevtree, "BUILD"))
+
+ def retry():
+ global stdout, stderr
+ spec_content = spec.render()
+ with open(spec_file, "w") as fileobj:
+ fileobj.write(spec_content)
+ rc, stdout, stderr = execute("rpmbuild", "-ba", spec_file, fatal=False)
+ if rc:
+ print(stdout)
+ print(stderr)
+ return False
+ rc, stdout, stderr = execute(
+ "sudo", "dnf", "install", "-y", rpm_file, fatal=False)
+ if rc:
+ os.unlink(rpm_file)
+ print(stdout)
+ print(stderr)
+ return False
+ return True
+
+ rc, stdout, stderr = execute("rpmbuild", "-bb", spec_file, fatal=False)
+ if rc:
+ if "cd: package: No such file or directory" in (stdout + stderr):
+ src_fixup = os.listdir(os.path.join(rpmdevtree, "BUILD"))[0]
+ print("=> Fixing srcdir %s" % src_fixup)
+ PACKAGE_SRC_DIR_NAME[metadata["manifest"]["name"]] = src_fixup
+ rc = not retry()
+ # if re.search("error: Bad exit status from .* .%check.", stdout):
+ if rc:
+ print("=> Retrying without test...")
+ spec.add_check = False
+ if not retry():
+ print(stderr)
+ print(stdout)
+ raise RuntimeError("%s: couldn't build" % spec_file)
+
+ rc, stdout, stderr = execute(
+ "sudo", "dnf", "install", "-y", rpm_file, fatal=False)
+ if rc:
+ os.unlink(rpm_file)
+ # Check for dep fixup
+ dep_fixup = []
+ src_fixup = None
+ for line in stderr.split('\n') + stdout.split('\n'):
+ if "nothing provides" in line:
+ dep_fixup.append(line.split()[3][5:-1])
+
+ if dep_fixup:
+ print("=> Retrying with fixdep %s" % dep_fixup)
+ spec.fixdep = dep_fixup
+ if not retry():
+ raise RuntimeError("%s: fixup didn't work" % stderr)
+
+ else:
+ print("stderr\n", stderr)
+ print("stdout\n", stdout)
+ os.unlink(rpm_file)
+ raise RuntimeError("%s: couldn't install" % rpm_file)
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..dca9a90
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+click
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..7d6ab5a
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,41 @@
+[metadata]
+name = npmfed
+summary = Npm packaging tool
+description-file = README.rst
+requires-python = >=3.5
+author = Tristan Cacqueray
+author-email = tdecacqu@redhat.com
+home-page = https://npmfed.softwarefactory-project.io/
+classifier =
+ Intended Audience :: Developers
+ Intended Audience :: System Administrators
+ License :: OSI Approved :: Apache Software License
+ Programming Language :: Python
+ Programming Language :: Python :: 3.5
+
+keywords = fedora, packaging, yarn, npm
+
+[tool:pytest]
+addopts = --verbose
+python_files = npmfed/tests/*.py
+
+[files]
+packages = npmfed
+
+[global]
+setup-hooks = pbr.hooks.setup_hook
+
+[build_sphinx]
+build-dir = build/doc/
+source-dir = doc/
+all_files = 1
+
+[upload_sphinx]
+upload-dir = build/doc/html
+
+[wheel]
+universal = 1
+
+[entry_points]
+console_scripts =
+ npmfed = npmfed.cmd:cli
diff --git a/setup.py b/setup.py
new file mode 100755
index 0000000..b0bea1e
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,5 @@
+import setuptools
+
+setuptools.setup(
+ setup_requires=['pbr'],
+ pbr=True)
diff --git a/test-requirements.txt b/test-requirements.txt
new file mode 100644
index 0000000..e079f8a
--- /dev/null
+++ b/test-requirements.txt
@@ -0,0 +1 @@
+pytest
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..2da8c09
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,18 @@
+[tox]
+envlist = py35,py36,pep8
+minversion = 1.6
+skipsdist = True
+
+[testenv]
+usedevelop = True
+deps = -rtest-requirements.txt
+commands = py.test -v
+passenv = LC_ALL
+
+[testenv:pep8]
+basepython = python3
+deps = flake8
+commands = flake8 --ignore=E26,E501,E251,E225,E722 npmfed
+
+[testenv:venv]
+commands = {posargs}