New catalog generation automatically pulls deps from GitHub

catalog.json

       }
     },
     "neo-concepts": {
-      "0.1.0": {
-        "depends": [],
-        "description": "Minimal C++ concepts library. Contains many definitions from C++20.",
-        "git": {
-          "ref": "0.1.0",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/neo-concepts.git"
-        }
-      },
-      "0.2.0": {
-        "depends": [],
-        "description": "Minimal C++ concepts library. Contains many definitions from C++20.",
-        "git": {
-          "ref": "0.2.0",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/neo-concepts.git"
-        }
-      },
-      "0.2.1": {
-        "depends": [],
-        "description": "Minimal C++ concepts library. Contains many definitions from C++20.",
-        "git": {
-          "ref": "0.2.1",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/neo-concepts.git"
-        }
-      },
       "0.2.2": {
         "depends": [],
-        "description": "Minimal C++ concepts library. Contains many definitions from C++20.",
+        "description": "A (mostly) backport of C++20 concepts library, with some additions.",
         "git": {
           "ref": "0.2.2",
           "transform": [],
       },
       "0.3.0": {
         "depends": [],
-        "description": "Minimal C++ concepts library. Contains many definitions from C++20.",
+        "description": "A (mostly) backport of C++20 concepts library, with some additions.",
         "git": {
           "ref": "0.3.0",
           "transform": [],
       },
       "0.3.1": {
         "depends": [],
-        "description": "Minimal C++ concepts library. Contains many definitions from C++20.",
+        "description": "A (mostly) backport of C++20 concepts library, with some additions.",
         "git": {
           "ref": "0.3.1",
           "transform": [],
       },
       "0.3.2": {
         "depends": [],
-        "description": "Minimal C++ concepts library. Contains many definitions from C++20.",
+        "description": "A (mostly) backport of C++20 concepts library, with some additions.",
         "git": {
           "ref": "0.3.2",
           "transform": [],
       }
     },
     "neo-fun": {
-      "0.1.0": {
-        "depends": [],
-        "description": "Some library fundamentals that you might find useful",
-        "git": {
-          "ref": "0.1.0",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/neo-fun.git"
-        }
-      },
       "0.1.1": {
         "depends": [],
-        "description": "Some library fundamentals that you might find useful",
+        "description": "Some library components that didn't quite fit anywhere else...",
         "git": {
           "ref": "0.1.1",
           "transform": [],
       },
       "0.2.0": {
         "depends": [],
-        "description": "Some library fundamentals that you might find useful",
+        "description": "Some library components that didn't quite fit anywhere else...",
         "git": {
           "ref": "0.2.0",
           "transform": [],
       },
       "0.2.1": {
         "depends": [],
-        "description": "Some library fundamentals that you might find useful",
+        "description": "Some library components that didn't quite fit anywhere else...",
         "git": {
           "ref": "0.2.1",
           "transform": [],
       },
       "0.3.0": {
         "depends": [],
-        "description": "Some library fundamentals that you might find useful",
+        "description": "Some library components that didn't quite fit anywhere else...",
         "git": {
           "ref": "0.3.0",
           "transform": [],
       },
       "0.3.1": {
         "depends": [],
-        "description": "Some library fundamentals that you might find useful",
+        "description": "Some library components that didn't quite fit anywhere else...",
         "git": {
           "ref": "0.3.1",
           "transform": [],
       },
       "0.3.2": {
         "depends": [],
-        "description": "Some library fundamentals that you might find useful",
+        "description": "Some library components that didn't quite fit anywhere else...",
         "git": {
           "ref": "0.3.2",
           "transform": [],
           "url": "https://github.com/vector-of-bool/neo-fun.git"
         }
-      }
-    },
-    "neo-sqlite3": {
-      "0.1.0": {
-        "depends": [],
-        "description": "A modern and low-level C++ SQLite API",
-        "git": {
-          "ref": "0.1.0",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/neo-sqlite3.git"
-        }
-      },
-      "0.2.0": {
-        "depends": [],
-        "description": "A modern and low-level C++ SQLite API",
-        "git": {
-          "ref": "0.2.0",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/neo-sqlite3.git"
-        }
       },
-      "0.2.1": {
+      "0.4.0": {
         "depends": [],
-        "description": "A modern and low-level C++ SQLite API",
+        "description": "Some library components that didn't quite fit anywhere else...",
         "git": {
-          "ref": "0.2.1",
+          "ref": "0.4.0",
           "transform": [],
-          "url": "https://github.com/vector-of-bool/neo-sqlite3.git"
-        }
-      },
-      "0.2.2": {
-        "depends": [],
-        "description": "A modern and low-level C++ SQLite API",
-        "git": {
-          "ref": "0.2.2",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/neo-sqlite3.git"
+          "url": "https://github.com/vector-of-bool/neo-fun.git"
         }
-      },
+      }
+    },
+    "neo-sqlite3": {
       "0.2.3": {
         "depends": [],
         "description": "A modern and low-level C++ SQLite API",
       }
     },
     "pubgrub": {
-      "0.1.2": {
-        "depends": [],
-        "description": "A C++ implementation of the Pubgrub version solving algorithm",
-        "git": {
-          "ref": "0.1.2",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/pubgrub.git"
-        }
-      },
-      "0.2.0": {
-        "depends": [],
-        "description": "A C++ implementation of the Pubgrub version solving algorithm",
-        "git": {
-          "ref": "0.2.0",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/pubgrub.git"
-        }
-      },
       "0.2.1": {
         "depends": [],
-        "description": "A C++ implementation of the Pubgrub version solving algorithm",
+        "description": "Pubgrub dependency resolution algorithm for C++",
         "git": {
           "ref": "0.2.1",
           "transform": [],
           "url": "https://github.com/ericniebler/range-v3.git"
         }
       },
+      "0.11.0": {
+        "depends": [],
+        "description": "Range library for C++14/17/20, basis for C++20's std::ranges",
+        "git": {
+          "auto-lib": "range-v3/range-v3",
+          "ref": "0.11.0",
+          "transform": [],
+          "url": "https://github.com/ericniebler/range-v3.git"
+        }
+      },
       "0.5.0": {
         "depends": [],
         "description": "Range library for C++14/17/20, basis for C++20's std::ranges",
       }
     },
     "semver": {
-      "0.2.1": {
-        "depends": [],
-        "description": "A C++ library that implements Semantic Versioning parsing, emitting, types, ordering, and operations. See https://semver.org/",
-        "git": {
-          "ref": "0.2.1",
-          "transform": [],
-          "url": "https://github.com/vector-of-bool/semver.git"
-        }
-      },
       "0.2.2": {
         "depends": [],
-        "description": "A C++ library that implements Semantic Versioning parsing, emitting, types, ordering, and operations. See https://semver.org/",
+        "description": "A C++ Library for Dealing with Semantic Versioning",
         "git": {
           "ref": "0.2.2",
           "transform": [],
     "vob-json5": {
       "0.1.5": {
         "depends": [],
-        "description": "A C++ implementation of a JSON5 parser",
+        "description": "A json5 library for C++",
         "git": {
           "ref": "0.1.5",
           "transform": [],
           "neo-fun^0.1.0",
           "neo-concepts^0.2.1"
         ],
-        "description": "A C++ library to process recursive dynamic data",
+        "description": "A generic library for dealing with semistructured data",
         "git": {
           "ref": "0.1.0",
           "transform": [],
           "neo-fun^0.1.1",
           "neo-concepts^0.2.2"
         ],
-        "description": "A C++ library to process recursive dynamic data",
+        "description": "A generic library for dealing with semistructured data",
         "git": {
           "ref": "0.1.1",
           "transform": [],
           "neo-fun^0.3.2",
           "neo-concepts^0.3.2"
         ],
-        "description": "A C++ library to process recursive dynamic data",
+        "description": "A generic library for dealing with semistructured data",
         "git": {
           "ref": "0.2.0",
           "transform": [],
           "neo-fun^0.3.2",
           "neo-concepts^0.3.2"
         ],
-        "description": "A C++ library to process recursive dynamic data",
+        "description": "A generic library for dealing with semistructured data",
         "git": {
           "ref": "0.2.1",
           "transform": [],

package.jsonc

     "depends": {
         "spdlog": "1.7.0",
         "ms-wil": "2020.3.16",
-        "range-v3": "0.10.0",
+        "range-v3": "0.11.0",
         "nlohmann-json": "3.7.1",
         "neo-sqlite3": "0.2.3",
         "neo-fun": "0.3.2",

tools/gen-catalog-json.py

 import argparse
+import os
 import json
+import json5
 import itertools
-from typing import NamedTuple, Tuple, List, Sequence, Union, Optional, Mapping
+import base64
+from urllib import request, parse as url_parse
+from typing import NamedTuple, Tuple, List, Sequence, Union, Optional, Mapping, Iterable
+import re
 from pathlib import Path
 import sys
 import textwrap
+from concurrent.futures import ThreadPoolExecutor
 
 
 class CopyMoveTransform(NamedTuple):
 class Version(NamedTuple):
     version: str
     remote: RemoteInfo
-    depends: Mapping[str, str] = {}
+    depends: Sequence[str] = []
     description: str = '(No description provided)'
 
     def to_dict(self) -> dict:
         ret: dict = {
             'description': self.description,
-            'depends': [k + v for k, v in self.depends.items()],
+            'depends': list(self.depends),
         }
         if isinstance(self.remote, Git):
             ret['git'] = self.remote.to_dict()
 
 class VersionSet(NamedTuple):
     version: str
-    depends: Sequence[Tuple[str, str]]
+    depends: Sequence[str]
 
 
 class Package(NamedTuple):
     versions: List[Version]
 
 
+HTTP_POOL = ThreadPoolExecutor(10)
+
+
+def github_http_get(url: str):
+    url_dat = url_parse.urlparse(url)
+    req = request.Request(url)
+    req.add_header('Accept-Encoding', 'application/json')
+    req.add_header('Authorization', f'token {os.environ["GITHUB_API_TOKEN"]}')
+    if url_dat.hostname != 'api.github.com':
+        raise RuntimeError(f'Request is outside of api.github.com [{url}]')
+    resp = request.urlopen(req)
+    if resp.status != 200:
+        raise RuntimeError(
+            f'Request to [{url}] failed [{resp.status} {resp.reason}]')
+    return json5.loads(resp.read())
+
+
+def _get_github_tree_file_content(url: str) -> bytes:
+    json_body = github_http_get(url)
+    content_b64 = json_body['content'].encode()
+    assert json_body['encoding'] == 'base64', json_body
+    content = base64.decodebytes(content_b64)
+    return content
+
+
+def _version_for_github_tag(pkg_name: str, desc: str, clone_url: str,
+                            tag) -> Version:
+    print(f'Loading tag {tag["name"]}')
+    commit = github_http_get(tag['commit']['url'])
+    tree = github_http_get(commit['commit']['tree']['url'])
+
+    tree_content = {t['path']: t for t in tree['tree']}
+    cands = ['package.json', 'package.jsonc', 'package.json5']
+    for cand in cands:
+        if cand in tree_content:
+            package_json_fname = cand
+            break
+    else:
+        raise RuntimeError(
+            f'No package JSON5 file in tag {tag["name"]} for {pkg_name} (One of {tree_content.keys()})'
+        )
+
+    package_json = json5.loads(
+        _get_github_tree_file_content(tree_content[package_json_fname]['url']))
+    version = package_json['version']
+    if pkg_name != package_json['name']:
+        raise RuntimeError(f'package name in repo "{package_json["name"]}" '
+                           f'does not match expected name "{pkg_name}"')
+
+    depends = package_json.get('depends')
+    pairs: Iterable[str]
+    if isinstance(depends, dict):
+        pairs = ((k + v) for k, v in depends.items())
+    elif isinstance(depends, list):
+        pairs = depends
+    elif depends is None:
+        pairs = []
+    else:
+        raise RuntimeError(
+            f'Unknown "depends" object from json file: {depends!r}')
+
+    remote = Git(url=clone_url, ref=tag['name'])
+    return Version(
+        version, description=desc, depends=list(pairs), remote=remote)
+
+
+def github_package(name: str, repo: str, want_tags: Iterable[str]) -> Package:
+    print(f'Downloading repo from {repo}')
+    repo_data = github_http_get(f'https://api.github.com/repos/{repo}')
+    desc = repo_data['description']
+    avail_tags = github_http_get(repo_data['tags_url'])
+
+    missing_tags = set(want_tags) - set(t['name'] for t in avail_tags)
+    if missing_tags:
+        raise RuntimeError(
+            'One or more wanted tags do not exist in '
+            f'the repository "{repo}" (Missing: {missing_tags})')
+
+    tag_items = (t for t in avail_tags if t['name'] in want_tags)
+
+    versions = HTTP_POOL.map(
+        lambda tag: _version_for_github_tag(name, desc, repo_data['clone_url'], tag),
+        tag_items)
+
+    return Package(name, list(versions))
+
+
 def simple_packages(name: str,
                     description: str,
                     git_url: str,
             description=description,
             remote=Git(
                 git_url, tag_fmt.format(ver.version), auto_lib=auto_lib),
-            depends={dep_name: dep_rng
-                     for dep_name, dep_rng in ver.depends}) for ver in versions
+            depends=ver.depends) for ver in versions
     ])
 
 
     ])
 
 
+# yapf: disable
 PACKAGES = [
+    github_package('neo-sqlite3', 'vector-of-bool/neo-sqlite3', ['0.2.3', '0.3.0']),
+    github_package('neo-fun', 'vector-of-bool/neo-fun', ['0.1.1', '0.2.0', '0.2.1', '0.3.0', '0.3.1', '0.3.2', '0.4.0']),
+    github_package('neo-concepts', 'vector-of-bool/neo-concepts', (
+            '0.2.2',
+            '0.3.0',
+            '0.3.1',
+            '0.3.2',
+        )),
+    github_package('semver', 'vector-of-bool/semver', ['0.2.2']),
+    github_package('pubgrub', 'vector-of-bool/pubgrub', ['0.2.1']),
+    github_package('vob-json5', 'vector-of-bool/json5', ['0.1.5']),
+    github_package('vob-semester', 'vector-of-bool/semester', ['0.1.0', '0.1.1', '0.2.0', '0.2.1']),
     many_versions(
         'magic_enum',
         (
             '0.9.0',
             '0.9.1',
             '0.10.0',
+            '0.11.0',
         ),
         git_url='https://github.com/ericniebler/range-v3.git',
         auto_lib='range-v3/range-v3',
             remote=Git('https://github.com/vector-of-bool/wil.git',
                        'dds/2020.03.16'))
     ]),
-    many_versions(
-        'neo-sqlite3',
-        (
-            '0.1.0',
-            '0.2.0',
-            '0.2.1',
-            '0.2.2',
-            '0.2.3',
-            '0.3.0',
-        ),
-        description='A modern and low-level C++ SQLite API',
-        git_url='https://github.com/vector-of-bool/neo-sqlite3.git',
-    ),
-    many_versions(
-        'neo-fun',
-        (
-            '0.1.0',
-            '0.1.1',
-            '0.2.0',
-            '0.2.1',
-            '0.3.0',
-            '0.3.1',
-            '0.3.2',
-        ),
-        description='Some library fundamentals that you might find useful',
-        git_url='https://github.com/vector-of-bool/neo-fun.git',
-    ),
-    many_versions(
-        'neo-concepts',
-        (
-            '0.1.0',
-            '0.2.0',
-            '0.2.1',
-            '0.2.2',
-            '0.3.0',
-            '0.3.1',
-            '0.3.2',
-        ),
-        description=
-        'Minimal C++ concepts library. Contains many definitions from C++20.',
-        git_url='https://github.com/vector-of-bool/neo-concepts.git',
-    ),
-    Package('semver', [
-        Version(
-            '0.2.1',
-            description=
-            'A C++ library that implements Semantic Versioning parsing, emitting, '
-            'types, ordering, and operations. See https://semver.org/',
-            remote=Git('https://github.com/vector-of-bool/semver.git',
-                       '0.2.1')),
-        Version(
-            '0.2.2',
-            description=
-            'A C++ library that implements Semantic Versioning parsing, emitting, '
-            'types, ordering, and operations. See https://semver.org/',
-            remote=Git('https://github.com/vector-of-bool/semver.git',
-                       '0.2.2')),
-    ]),
-    many_versions(
-        'pubgrub',
-        (
-            '0.1.2',
-            '0.2.0',
-            '0.2.1',
-        ),
-        description=
-        'A C++ implementation of the Pubgrub version solving algorithm',
-        git_url='https://github.com/vector-of-bool/pubgrub.git',
-    ),
-    many_versions(
-        'vob-json5',
-        ('0.1.5', ),
-        description='A C++ implementation of a JSON5 parser',
-        git_url='https://github.com/vector-of-bool/json5.git',
-    ),
-    simple_packages(
-        'vob-semester',
-        description='A C++ library to process recursive dynamic data',
-        git_url='https://github.com/vector-of-bool/semester.git',
-        versions=[
-            VersionSet('0.1.0', [
-                ('neo-fun', '^0.1.0'),
-                ('neo-concepts', '^0.2.1'),
-            ]),
-            VersionSet('0.1.1', [
-                ('neo-fun', '^0.1.1'),
-                ('neo-concepts', '^0.2.2'),
-            ]),
-            VersionSet('0.2.0', [
-                ('neo-fun', '^0.3.2'),
-                ('neo-concepts', '^0.3.2'),
-            ]),
-            VersionSet('0.2.1', [
-                ('neo-fun', '^0.3.2'),
-                ('neo-concepts', '^0.3.2'),
-            ]),
-        ],
-    ),
     many_versions(
         'ctre',
         (
             Version(
                 ver,
                 description='Fast C++ logging library',
-                depends={'fmt': '+6.0.0'},
+                depends=['fmt+6.0.0'],
                 remote=Git(
                     url='https://github.com/gabime/spdlog.git',
                     ref=f'v{ver}',
             ver,
             description=
             'A C++ <-> Lua API wrapper with advanced features and top notch performance',
-            depends={'lua': '+0.0.0'},
+            depends=['lua+0.0.0'],
             remote=Git(
                 'https://github.com/ThePhD/sol2.git',
                 f'v{ver}',
         *(Version(
             ver,
             description='A Template Engine for Modern C++',
-            depends={'nlohmann-json': '+0.0.0'},
+            depends=['nlohmann-json+0.0.0'],
             remote=Git(
                 'https://github.com/pantor/inja.git',
                 f'v{ver}',
         transforms=[FSTransform(remove=RemoveTransform(path='src/'))],
     ),
 ]
+# yapf: enable
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()