rework and refactor

ncc

@@ -1,3 +0,0 @@
-#!/bin/env python3
-from nginx_configurator import main
-main.cli()

ncc.yml.sample

@@ -0,0 +1,8 @@
+conf_dir: "test-conf"
+main_config: "nginx.conf"
+dehydrated_dir: "dehydrated"
+sites_dir: "sites"
+
+targets:
+  - "ssh-host:/path/to/deployment"
+  - "/path/to/local/deployment"

ncc/__main__.py

@@ -0,0 +1,3 @@
+from .main import cli
+
+cli.main()

ncc/certs.py

@@ -0,0 +1,131 @@
+import glob
+import os
+import re
+import sys
+from collections import deque
+from collections.abc import Generator
+from pathlib import Path
+from string import Template
+from typing import Tuple
+
+from .error import NccException
+
+SSL_CONFIG_TEMPLATE = """
+ssl_certificate_key $keypath;
+ssl_certificate $certpath;
+""".lstrip()
+SERVER_BLOCK_RE = re.compile(r"(?:^|[{};])\s*server\s*{", re.MULTILINE)
+INCLUDE_RE = re.compile(r"(?:^|[{};])\s*include\s+([^;]+);", re.MULTILINE)
+SERVER_NAME_RE = re.compile(r"(?:^|[{};])\s*server_name\s+([^;]+);", re.MULTILINE)
+
+
+class ConfigError(Exception):
+    pass
+
+
+def _config_files(main_cfg: Path) -> Generator[Tuple[Path, str]]:
+    cfg = main_cfg
+    text = cfg.read_text()
+    yield (cfg, text)
+
+    for include in re.finditer(INCLUDE_RE, text):
+        for file in glob.glob(include.group(1)):
+            for cfg in _config_files(Path(file)):
+                yield cfg
+
+
+def _remove_comments(cfg: str) -> str:
+    """I haven't actually read the parser, so we might be discarding more than we should, oh well"""
+    output = []
+    for line in cfg.splitlines():
+        output.append(line.split("#")[0])
+
+    return "\n".join(output)
+
+
+def _get_server_blocks(cfg: str) -> Generator[str]:
+    for server_block_start in re.finditer(SERVER_BLOCK_RE, cfg):
+        start_idx = idx = server_block_start.end()
+        brackets = 1
+
+        while brackets > 0 and idx < len(cfg):
+            if cfg[idx] == "{":
+                brackets += 1
+            elif cfg[idx] == "}":
+                brackets -= 1
+            idx += 1
+
+        yield cfg[start_idx : idx - 1]
+
+
+def get_sites(cfg: Path) -> Generator[Tuple[Path, str]]:
+    """Expects to be in the conf dir"""
+
+    for c in _config_files(cfg):
+        config_part = _remove_comments(c[1])
+
+        for server_block in _get_server_blocks(config_part):
+            sn = next(re.finditer(SERVER_NAME_RE, server_block))
+            if not sn:
+                continue
+
+            domains = sn.group(1).split()
+
+            for domain in domains:
+                yield (c[0], domain)
+
+
+def generate_ssl(cfg: Path, domainstxt_file: Path) -> int:
+    """Expects to be in the conf dir"""
+
+    to_generate = {}
+
+    os.makedirs("genssl/", exist_ok=True)
+    os.makedirs(domainstxt_file.parent, exist_ok=True)
+
+    for c in _config_files(cfg):
+        config_part = _remove_comments(c[1])
+
+        for server_block in _get_server_blocks(config_part):
+            sn = next(re.finditer(SERVER_NAME_RE, server_block))
+            if not sn:
+                continue
+
+            wants_ssl = next(
+                (
+                    i.group(1)
+                    for i in INCLUDE_RE.finditer(server_block)
+                    if i.group(1).startswith("genssl/")
+                ),
+                None,
+            )
+
+            domains = sn.group(1).split()
+            domains.sort()
+
+            if wants_ssl:
+                if (entry := to_generate.get(wants_ssl)) and entry != domains:
+                    raise NccException(
+                        f'config requests "{wants_ssl}" with different domain sets'
+                    )
+                else:
+                    to_generate[wants_ssl] = domains
+
+    domainstxt = []
+    for file, domains in to_generate.items():
+        file_no_prefix = file.removeprefix("genssl/")
+        Path(file).write_text(
+            Template(SSL_CONFIG_TEMPLATE).substitute(
+                {
+                    "keypath": domainstxt_file.parent
+                    / f"certs/{file_no_prefix}/privkey.pem",
+                    "certpath": domainstxt_file.parent
+                    / f"certs/{file_no_prefix}/fullchain.pem",
+                }
+            )
+        )
+        domainstxt.append(" ".join(domains) + " > " + file_no_prefix + "\n")
+
+    domainstxt_file.write_text("".join(domainstxt))
+
+    return 0

ncc/error.py

@@ -0,0 +1,13 @@
+import click
+import sys
+
+
+class NccException(click.ClickException):
+    def __init__(self, *args, log=None):
+        super().__init__(*args)
+        self.log = log
+
+    def show(self, file=None) -> None:
+        if self.log:
+            click.echo(self.log)
+        click.secho("ERROR: " + self.format_message(), file or sys.stderr, fg="red")

ncc/main.py

@@ -0,0 +1,264 @@
+import os
+import tempfile
+from collections import defaultdict
+from pathlib import Path
+
+import click
+import sysrsync
+import yaml
+
+from . import certs, sysaction, templating
+from .error import NccException
+
+CONFIG = {
+    "main_config": "nginx.conf",
+    "conf_dir": ".",
+    "targets": [],
+    "dehydrated_dir": "dehydrated",
+    "sites_dir": "sites",
+}
+
+
+class Step:
+    """Fancy printing of steps with option to hide the details after it finishes"""
+
+    def __init__(self, text, hide=False):
+        self.text = text
+        self.lines = 1
+        self.on_start = True
+        self.hide = hide
+
+    def echo(self, message, *args, **kwargs) -> None:
+        nl = int(kwargs.get("nl", 1))
+        self.lines += message.count("\n") + nl
+
+        if self.on_start:
+            click.echo("  => ", nl=False)
+        self.on_start = nl or (bool(message) and message[-1] == "\n")
+
+        click.echo("\n  => ".join(message.splitlines()), *args, **kwargs)
+
+    def __enter__(self) -> "Step":
+        click.secho("[ ] " + self.text)
+        return self
+
+    def __exit__(self, exc, _exc2, _exc3) -> None:
+        click.echo(
+            f'\r\x1b[{self.lines}F[{click.style("K" if not exc else "E", fg="green" if not exc else "red")}'
+        )
+        if self.hide and not exc:
+            click.echo("\x1b[0J", nl=False)
+        elif self.lines > 1:
+            click.echo(f"\x1b[{self.lines-1}E", nl=False)
+
+
+def load_config():
+    cwd = Path(os.getcwd())
+    conf_file = None
+
+    root = Path(cwd.root)
+    while cwd != root:
+        conf_file = cwd / "ncc.yml"
+        if conf_file.exists():
+            os.chdir(cwd)
+            break
+        else:
+            conf_file = None
+        cwd = cwd.parent
+
+    if not conf_file:
+        click.echo(
+            "Failed to find ncc configuration (searched all parent folders)", err=True
+        )
+        exit(1)
+
+    with conf_file.open() as f:
+        if yml := yaml.safe_load(f):
+            CONFIG.update(yml)
+
+    os.chdir(CONFIG["conf_dir"])
+
+
+@click.group()
+def cli():
+    """Update the nginx cluster configuration
+
+    MUST BE RAN ON MASTER (will detect automatically)
+    """
+
+    load_config()
+
+
+@cli.command()
+@click.option(
+    "--dehydrated-only",
+    type=bool,
+    is_flag=True,
+    help="Only fetches and deploys new certificates",
+)
+@click.option("--skip-master-check", type=bool, is_flag=True)
+def up(dehydrated_only: bool, skip_master_check: bool):
+    """Deploy the configuration to the cluster
+
+    Does the following:
+
+      1. generates ssl config in genssl folder
+
+      2. runs dehydrated
+
+      3. checks the configuration
+
+      4. deploys to the cluster
+    """
+
+    if not skip_master_check and not sysaction.is_keepalived_master():
+        click.echo("Refusing to start. Not running on master.", err=True)
+        exit(1)
+
+    with Step("Preparing configuration..."):
+        dehydrated_dir = Path(CONFIG["dehydrated_dir"])
+        if not dehydrated_only:
+            certs.generate_ssl(
+                Path(CONFIG["main_config"]),
+                Path(CONFIG["dehydrated_dir"]) / "domains.txt",
+            )
+
+        ec, stdout = sysaction.run_shell(
+            (str(dehydrated_dir / "dehydrated.sh"), "-c"), window_height=5
+        )
+        if ec != 0:
+            log = Path(tempfile.mktemp())
+            log.write_text(stdout)
+            raise NccException(
+                f"dehydrated returned {ec} (log: {click.format_filename(log)})"
+            )
+
+        ec, stdout = sysaction.run_shell(
+            ("nginx", "-t", "-p", ".", "-c", CONFIG["main_config"]), window_height=5
+        )
+        if ec != 0:
+            raise NccException("configuration did not pass nginx test", log=stdout)
+
+    with Step("Deploying to cluster...") as step:
+        for target in CONFIG["targets"]:
+            step.echo('deploying to "' + target + '"', nl=False)
+
+            try:
+                sysrsync.run(
+                    source=os.getcwd(),
+                    destination=target,
+                    exclusions=[str(dehydrated_dir / "archive/**")],
+                    options=["-a", "--delete"],
+                )
+            except Exception as e:
+                step.echo(click.style(" E", fg="red"))
+                raise NccException("failed to rsync configuration") from e
+
+            # technically we could be misinterpreting a path here...
+            if (first_part := target.split("/")[0]) and (
+                remote := first_part.split(":")[0]
+            ):
+                ec, stdout = sysaction.run_shell(("ssh", "-T", remote, "echo 1"))
+            else:
+                ec, stdout = sysaction.run_shell(("echo", "1"))
+
+            if ec != 0:
+                step.echo(click.style(" E", fg="red"))
+                raise NccException("failed to reload nginx", log=stdout)
+
+            step.echo(click.style(" K", fg="green"))
+
+
+@cli.command()
+def test():
+    """Run nginx -t on the configuration"""
+
+    ec, stdout = sysaction.run_shell(
+        ("nginx", "-t", "-p", ".", "-c", CONFIG["main_config"])
+    )
+    click.echo(stdout, nl=False)
+    exit(ec)
+
+
+def shell_complete_service(_ctx, _param, incomplete):
+    load_config()
+    return [
+        site
+        for _, site in certs.get_sites(Path(CONFIG["main_config"]))
+        if incomplete in site
+    ]
+
+
+@cli.command()
+@click.argument("site", shell_complete=shell_complete_service)
+def edit(site: str):
+    """Edit a site"""
+
+    file = next(
+        (f for f, s in certs.get_sites(Path(CONFIG["main_config"])) if site == s), None
+    )
+    if not file or not file.exists():
+        raise NccException("could not find site")
+
+    click.edit(filename=str(file))
+
+
+@cli.command("list")
+def list_():
+    """List all sites and the files they are located in"""
+
+    file_sites = defaultdict(list)
+
+    for f, s in certs.get_sites(Path(CONFIG["main_config"])):
+        file_sites[f].append(s)
+
+    for file, sites in file_sites.items():
+        click.echo(str(file) + ": " + " ".join(sites))
+
+
+@cli.command()
+def new():
+    """Create a new site"""
+
+    sites_dir = Path(CONFIG["sites_dir"])
+    if not sites_dir.exists():
+        raise NccException("sites_dir does not exist")
+
+    templates = [f.name for f in Path("templates").glob("*.conf")]
+    templates.sort()
+
+    printed = len(templates)
+    for i, t in enumerate(templates):
+        click.echo(f" {i+1}) {t}")
+
+    while True:
+        try:
+            printed += 1
+            choice = int(input("Template number: ")) - 1
+
+        except ValueError:
+            continue
+        if 0 <= choice < len(templates):
+            break
+    click.echo(f"\r\x1b[{printed}F\x1b[0J", nl=False)
+    click.secho(f"=== {templates[choice]}", bold=True)
+
+    name, filled = templating.fill_template(
+        Path(f"templates/{templates[choice]}").read_text()
+    )
+
+    if not name:
+        name = input("File name (without .conf): ").removesuffix(".conf")
+
+    while not name or (file := sites_dir / (name + ".conf")).exists():
+        if name and click.confirm(
+            f"File {click.format_filename(file)} already exists. Overwrite?"
+        ):
+            break
+        name = input("File name (without .conf): ").removesuffix(".conf")
+
+    file.write_text(filled)
+    click.echo(f"Site written to {click.format_filename(file)}")
+
+    if click.confirm("Continue editing?"):
+        click.edit(filename=str(file))

ncc/sysaction.py

@@ -0,0 +1,64 @@
+import os
+import signal
+import subprocess
+import sys
+import time
+from collections import deque
+from pathlib import Path
+
+import click
+import sysrsync
+
+
+def run_shell(cmd, window_height=0):
+    out = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+    )
+    stdout = []
+    if window_height:
+        h = window_height
+        try:
+            buf = deque()
+            for line in out.stdout:
+                term_size = os.get_terminal_size()
+                prefixed = ("  => " + line).rstrip()[: term_size[0] - 1] + "\n"
+                stdout.append(line)
+                buf.append(prefixed)
+
+                if h != 0:
+                    sys.stdout.write(prefixed)
+                    sys.stdout.flush()
+                    h -= 1
+                    continue
+
+                buf.popleft()
+                sys.stdout.write(f"\x1b[{window_height}F\x1b[0J")
+                sys.stdout.write("".join(buf))
+                sys.stdout.flush()
+        finally:
+            if h < window_height:
+                sys.stdout.write(f"\x1b[{window_height-h}F\x1b[0J")
+            sys.stdout.flush()
+
+    out.wait()
+
+    return out.returncode, "".join(stdout) if stdout else out.stdout.read()
+
+
+def is_keepalived_master():
+    keepalived_data = Path("/tmp/keepalived.data")
+    keepalived_data.unlink(missing_ok=True)  # do not read old data
+
+    sig = int(os.getenv("KEEPALIVED_DATA_SIGNAL", signal.SIGUSR1))
+    os.kill(int(Path("/run/keepalived.pid").read_text()), sig)
+
+    # wait for keepalived
+    for _ in range(10):
+        time.sleep(0.05)
+        if keepalived_data.exists():
+            break
+    else:
+        raise Exception(f"keepalived did not produce data on signal {sig}")
+
+    # TODO: could we do better?
+    return "State = MASTER" in Path("/tmp/keepalived.data").read_text()

ncc/templating.py

@@ -0,0 +1,52 @@
+import bisect
+import re
+from collections import defaultdict
+from typing import Optional, Tuple
+
+IDENTIFIER_RE = re.compile(r"(?<!%)(?:%(\w+)\b|%{(\w+)})")
+
+
+def fill_template(template_str: str) -> Tuple[Optional[str], str]:
+    lines = template_str.splitlines()
+
+    identifier_contexts = defaultdict(list)
+
+    for i, line in enumerate(lines):
+        for match in re.finditer(IDENTIFIER_RE, line):
+            name = match[1] or match[2]
+            for j in range(max(0, i - 2), min(len(lines) - 1, i + 2)):
+                ctx = identifier_contexts[name]
+                idx = bisect.bisect_left(ctx, j)
+                if idx >= len(ctx) or ctx[idx] != j:
+                    ctx.insert(idx, j)
+
+    replacements = {}
+    for identifier, ctx in identifier_contexts.items():
+
+        def colorize(match):
+            name = match[1] or match[2]
+            if name == identifier:
+                return f"\x1b[31m%{match[1] or match[2]}\x1b[0m"
+            else:
+                return match[0]
+
+        prev = 0
+        printed = 0
+
+        for line_idx in ctx:
+            print(re.sub(IDENTIFIER_RE, colorize, lines[line_idx]))
+            printed += 1
+            if prev and line_idx - prev > 1:
+                print("\x1b[1m---\x1b[0m")
+                printed += 1
+            prev = line_idx
+
+        replacements[identifier] = input("\x1b[1m => " + identifier + ":\x1b[0m ")
+        printed += 1
+
+        print(f"\r\x1b[{printed}F\x1b[0J", end="")
+
+    def replace(match):
+        return replacements[match[1] or match[2]]
+
+    return replacements.get("name"), re.sub(IDENTIFIER_RE, replace, template_str).replace("%%","%")

nginx_configurator/certs.py

@@ -1,82 +0,0 @@
-import os
-import re
-from pathlib import Path
-from typing import List
-from .templating import jinja
-import glob
-
-DOMAINS_RE = re.compile(
-    r"^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})$"
-)
-
-
-def _walk_nginx_conf(nginx_conf: Path):
-    """Recursively finds all configuration files of a nginx config"""
-
-    # this path is not necessarily correct... someone could have a weird prefix and
-    # conf file combination
-    conf_dir = nginx_conf.parent
-
-    stack = [nginx_conf]
-
-    while len(stack):
-        file = stack.pop()
-        conf = file.read_text()
-
-        yield file, conf
-
-        for include in re.finditer(r"(?:^|\n\s*|[{;]\s*)include (.+);", conf):
-            pattern = include.group(1)
-            for file in glob.glob(
-                pattern if pattern.startswith("/") else f"{conf_dir}/pattern"
-            ):
-                stack.append(Path(file))
-
-
-def gather_autossl_directives(nginx_conf: Path):
-    """
-    Finds #AUTOSSL directives inside an nginx configuration. The server_name
-    must be on a separate line. (which it usually is)
-    """
-    directives = []
-    for _, conf in _walk_nginx_conf(nginx_conf):
-        for directive in re.finditer(
-            r"(?:^|\n\s*|[{;]\s*)server_name (.*); *# *AUTOSSL *> *(\S+)", conf
-        ):
-            domains, alias = directive.groups()
-            domains = domains.split()
-
-            if any(not re.match(DOMAINS_RE, domain) for domain in domains):
-                raise ValueError(
-                    f"Cannot get SSL cert for \"{''.join(domains)}\". Invalid domains."
-                )
-
-            if not re.match(r"^[a-zA-Z0-9-_]+$", alias):
-                raise ValueError(f'Invalid cert alias "{alias}"')
-
-            directives.append((domains, alias))
-
-    return directives
-
-
-def get_site_files(nginx_dir: Path) -> List[Path]:
-    names = []
-    for file in (nginx_dir / "sites").iterdir():
-        if file.is_file() and re.match(r"\d+-.+\.conf", file.name):
-            names.append(file)
-
-    return names
-
-
-def build_domains_txt(directives):
-    return (
-        "\n".join(" ".join(domains) + " > " + alias for domains, alias in directives)
-        + "\n"
-    )
-
-
-def generate_ssl_configs(dir: Path, cert_aliases: List[str]):
-    for alias in cert_aliases:
-        file = dir / (alias + ".conf")
-        template = jinja.get_template("ssl.conf")
-        file.write_text(template.render(alias=alias))

nginx_configurator/main.py

@@ -1,353 +0,0 @@
-import json
-import os
-import re
-import tempfile
-from pathlib import Path
-import click
-from dotenv import load_dotenv
-from . import sysaction, certs
-from .sysaction import quit_on_err
-from .templating import jinja
-
-load_dotenv(os.getenv("DOTENV_PATH", "/etc/ncc/env"))
-
-NGINX_DIR = Path(os.getenv("NGINX_DIR", "/etc/nginx"))
-DOMAINS_TXT = Path(os.getenv("DOMAINS_TXT", "/etc/dehydrated/domains.txt"))
-REMOTE = os.getenv("REMOTE")
-REMOTE_SSH_KEY = os.getenv("REMOTE_SSH_KEY")
-DEHYDRATED_BIN = os.getenv("DEHYDRATED_BIN", "dehydrated")
-DEHYDRATED_TRIGGER_FILE = Path(
-    os.getenv("DEHYDRATED_TRIGGER_FILE", "/tmp/ncc-ssl-trigger")
-)
-CLUSTERS_FILE = Path(os.getenv("CLUSTERS_FILE", "/etc/ncc/clusters.json"))
-
-
-@click.group()
-@click.option("--skip-master-check", type=bool, is_flag=True)
-def cli(skip_master_check: bool):
-    """Update the nginx cluster configuration
-
-    MUST BE RAN ON MASTER (will detect automatically)
-    """
-
-    if not skip_master_check and not sysaction.is_keepalived_master():
-        click.echo("Refusing to start. Not running on master.", err=True)
-        exit(1)
-
-
-@cli.command()
-def reload():
-    """Replicate the local config and reload the nginx cluster
-
-    Does the following:
-      1. checks the nginx configuration
-      2. generates domains.txt for dehydrated
-      3. runs dehydrated to obtain certificates
-      4. reloads nginx
-      5. replicates the validated configuration
-    """
-
-    # check nginx config
-    quit_on_err(
-        sysaction.check_nginx_config(NGINX_DIR),
-        print_always=True,
-        additional_info="Nginx configuration is incorrect",
-    )
-
-    # build domains.txt
-    try:
-        directives = certs.gather_autossl_directives(NGINX_DIR / "nginx.conf")
-
-        DOMAINS_TXT.write_text(certs.build_domains_txt(directives))
-    except ValueError as e:
-        click.secho(e, err=True, fg="red")
-        exit(1)
-
-    # obtain certs
-    quit_on_err(
-        sysaction.run_dehydrated(DEHYDRATED_BIN),
-        additional_info="Failed to run dehydrated",
-    )
-    certs.generate_ssl_configs(NGINX_DIR / "ssl", [d[1] for d in directives])
-
-    # reload nginx
-    quit_on_err(sysaction.reload_nginx(), additional_info="Failed to reload nginx")
-
-    # replicate to remote
-    sysaction.remote_replication(REMOTE, REMOTE_SSH_KEY)
-    quit_on_err(
-        sysaction.remote_check_nginx_config(REMOTE, REMOTE_SSH_KEY, NGINX_DIR),
-        additional_info="Remote nginx configuration is incorrect",
-    )
-    quit_on_err(
-        sysaction.remote_reload_nginx(REMOTE, REMOTE_SSH_KEY),
-        additional_info="Remote nginx failed to reload",
-    )
-
-
-def get_upstreams():
-    clusters = json.loads(Path(CLUSTERS_FILE).read_text())
-    click.echo("Existing clusters:")
-    line = ""
-    for cluster in clusters:
-        if len(line + " " + cluster["name"]) > 78:
-            click.echo("  " + line.strip())
-            line = cluster["name"]
-        else:
-            line += " " + cluster["name"]
-    click.echo("  " + line.strip())
-
-    upstreams = click.prompt("Comma-separated upstreams, or an existing cluster name")
-
-    cluster = next((c for c in clusters if c["name"] == upstreams), None)
-    if cluster:
-        upstreams = cluster["upstreams"]
-    else:
-        upstreams = [u.strip() for u in upstreams.split(",")]
-
-    return upstreams
-
-
-def get_id():
-    return max(int(n.name.split("-")[0]) for n in certs.get_site_files(NGINX_DIR)) + 1
-
-
-def test_config(config: str, file: Path):
-    old_config = None
-    if file.exists():
-        old_config = file.read_text()
-
-    if config:
-        file.write_text(config)
-    else:
-        file.unlink(missing_ok=True)
-
-    c, out = sysaction.check_nginx_config(NGINX_DIR)
-    if c:
-        try:
-            certs.gather_autossl_directives(NGINX_DIR / "nginx.conf")
-        except ValueError as e:
-            out = e
-            c = False
-
-    # rollback
-    if old_config is not None:
-        file.write_text(old_config)
-    else:
-        file.unlink()
-
-    return c, out
-
-
-def edit_service(config, service_file: Path):
-    ok = False
-
-    new_config = None
-    while not ok:
-        new_config = click.edit(config, extension=".conf")
-
-        if new_config is not None:
-            config = new_config
-
-        c, out = test_config(config, service_file)
-
-        if not c:
-            click.echo(out, err=True)
-            click.secho("Failed to verify configuration", fg="red")
-
-            choice = click.prompt(
-                "Edit service configuration?", type=click.Choice(("yes", "abort"))
-            )
-            if choice == "abort":
-                tmp = Path(tempfile.mktemp())
-                tmp.write_text(config)
-                click.echo(f"Unfinished service written to {tmp}")
-                break
-
-        ok = c
-
-    return new_config, ok
-
-
-@cli.command()
-@click.pass_context
-def new(ctx):
-    """Create a new service"""
-    config = ""
-    should_open_editor = False
-    id = get_id()
-
-    if click.confirm("Would you like to use the default template?", default=True):
-        template = jinja.get_template("nginx-site.conf")
-
-        domains = [
-            d.strip() for d in click.prompt("Comma-separated domains").split(",")
-        ]
-        upstreams = get_upstreams()
-        port = click.prompt("Upstream port", type=int)
-        proto = (
-            click.prompt(
-                "Upstream protocol",
-                type=click.Choice(("http", "https")),
-                show_choices=True,
-            )
-            + "://"
-        )
-
-        config = template.render(
-            id=id, domains=domains, upstreams=upstreams, port=port, proto=proto
-        )
-        # create ssl file so configuration test passes
-        Path(f"/etc/nginx/ssl/{id}.conf").touch()
-
-        filename = f"{id}-{domains[0]}.conf"
-    else:
-        should_open_editor = True  # force open the config
-        template = jinja.get_template("nginx-minimal.conf")
-        config = template.render(id=id)
-        filename = f'{id}-{click.prompt("Service name (used as filename (eg. 01-service.conf))")}.conf'
-
-    # XXX: beware of weird code below
-
-    should_open_editor = should_open_editor or click.confirm(
-        "Would you like to edit the config?", default=True
-    )
-
-    # assume the user wants to edit the file if we are opening the editor
-    service_file = Path(
-        NGINX_DIR / "sites" / filename
-    )
-
-    ok = False
-    while not ok:
-        if should_open_editor:
-            new_cfg, ok = edit_service(config, service_file)
-
-            if not ok:
-                break
-            if new_cfg:
-                config = new_cfg
-        else:
-            ok, out = test_config(config, service_file)
-            if not ok:
-                click.echo(out, err=True)
-                click.secho("Failed to verify nginx configuration", fg="red")
-                choice = click.prompt(
-                    "Edit service configuration?", type=click.Choice(("yes", "abort"))
-                )
-                if choice == "abort":
-                    break
-                should_open_editor = True
-
-    if not ok:
-        Path(f"/etc/nginx/ssl/{id}.conf").unlink(missing_ok=True)
-        exit(1)
-
-    service_file.write_text(config)
-
-    ctx.invoke(reload)
-
-
-def shell_complete_service(ctx, param, incomplete):
-    return [file.name for file in certs.get_site_files(NGINX_DIR) if incomplete in file.name]
-
-@cli.command()
-@click.argument("service", shell_complete=shell_complete_service)
-@click.pass_context
-def edit(ctx, service: str):
-    """Edit a service"""
-    filename = service if service.endswith(".conf") else service + ".conf"
-
-    file = NGINX_DIR / "sites" / filename
-
-    if not file.exists():
-        click.secho(f"Service {filename} does not exist", fg="red")
-        exit(1)
-
-    config = file.read_text()
-    new_cfg, ok = edit_service(config, file)
-
-    if not new_cfg:
-        exit(0)
-
-    if not ok:
-        exit(1)
-
-    file.write_text(new_cfg)
-
-    ctx.invoke(reload)
-
-
-@cli.command()
-@click.argument("service", shell_complete=shell_complete_service)
-@click.pass_context
-def delete(ctx, service: str):
-    """Delete a service"""
-    filename = service if service.endswith(".conf") else service + ".conf"
-
-    file = NGINX_DIR / "sites" / filename
-    if not file.exists():
-        click.secho(f"Service {filename} does not exist", fg="red")
-        exit(1)
-
-    c, out = test_config("", file)
-    if not c:
-        click.echo(out, err=True)
-        click.secho("Failed to verify nginx configuration", fg="red")
-        click.echo("Service was not deleted")
-        exit(1)
-
-    file.unlink()
-
-    ctx.invoke(reload)
-
-
-@cli.command("list")
-def list_():
-    """List exsiting services and domain names associated with them"""
-    files = certs.get_site_files(NGINX_DIR)
-    for file in files:
-        config = file.read_text()
-        domain_names = set()
-        for directive in re.finditer(r"(?:^|\n\s*|[{;]\s*)server_name (.*);", config):
-            for domain in directive.group(1).split():
-                domain_names.add(domain)
-
-        click.echo(f"{file.name}: {' '.join(domain_names)}")
-
-
-@cli.command()
-def autossl():
-    """Renew SSL certificates and replicate changes"""
-    # build domains.txt
-    try:
-        directives = certs.gather_autossl_directives(NGINX_DIR / "nginx.conf")
-
-        DOMAINS_TXT.write_text(certs.build_domains_txt(directives))
-    except ValueError as e:
-        click.secho(e, err=True, fg="red")
-        exit(1)
-
-    # obtain certs
-    quit_on_err(
-        sysaction.run_dehydrated(DEHYDRATED_BIN),
-        additional_info="Failed to run dehydrated",
-    )
-
-    if DEHYDRATED_TRIGGER_FILE.exists():
-        click.echo("Certificates changed - reloading cluster")
-        certs.generate_ssl_configs(NGINX_DIR / "ssl", [d[1] for d in directives])
-
-        # reload nginx
-        quit_on_err(sysaction.reload_nginx(), additional_info="Failed to reload nginx")
-
-        # replicate to remote
-        sysaction.remote_replication(REMOTE, REMOTE_SSH_KEY)
-        quit_on_err(
-            sysaction.remote_check_nginx_config(REMOTE, REMOTE_SSH_KEY, NGINX_DIR),
-            additional_info="Remote nginx configuration is incorrect",
-        )
-        quit_on_err(
-            sysaction.remote_reload_nginx(REMOTE, REMOTE_SSH_KEY),
-            additional_info="Remote nginx failed to reload",
-        )

nginx_configurator/sysaction.py

@@ -1,85 +0,0 @@
-import os
-from pathlib import Path
-import signal
-import subprocess
-import time
-import click
-import sysrsync
-
-
-def _run_shell(cmd):
-    out = subprocess.run(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
-    )
-    return out.returncode == 0, out.stdout
-
-
-def quit_on_err(out, print_always=False, additional_info=""):
-    if (not out[0] or print_always) and len(out[1].strip()) > 0:
-        # print out nginx warnings
-        click.echo(out[1], err=True)
-
-    if not out[0]:
-        if additional_info:
-            click.secho(additional_info, err=True, fg="red")
-        exit(1)
-
-
-def is_keepalived_master():
-    keepalived_data = Path("/tmp/keepalived.data")
-    keepalived_data.unlink(missing_ok=True)  # do not read old data
-
-    sig = int(os.getenv("KEEPALIVED_DATA_SIGNAL", signal.SIGUSR1))
-    os.kill(int(Path("/run/keepalived.pid").read_text()), sig)
-
-    # wait for keepalived
-    for _ in range(10):
-        time.sleep(0.05)
-        if keepalived_data.exists():
-            break
-    else:
-        raise Exception(f"keepalived did not produce data on signal {sig}")
-
-    # TODO: could we do better?
-    return "State = MASTER" in Path("/tmp/keepalived.data").read_text()
-
-
-def reload_nginx():
-    return _run_shell(("nginx", "-s", "reload"))
-
-
-def check_nginx_config(nginx_dir: Path):
-    return _run_shell(("nginx", "-t", "-c", str(nginx_dir / "nginx.conf")))
-
-
-def run_dehydrated(dehydrated_bin: str):
-    return _run_shell((dehydrated_bin, "-c"))
-
-
-def remote_replication(remote, ssh_key):
-    # Copy nginx config to second server
-    sysrsync.run(
-        source="/etc/nginx/",
-        destination="/etc/nginx/",
-        destination_ssh=remote,
-        private_key=ssh_key,
-        options=["-a", "--delete"],
-    )
-    # Copy certificates to second server
-    sysrsync.run(
-        source="/etc/autossl/",
-        destination="/etc/autossl/",
-        destination_ssh=remote,
-        private_key=ssh_key,
-        options=["-a", "--delete"],
-    )
-
-
-def remote_check_nginx_config(remote, ssh_key, nginx_dir: Path):
-    # Check and reload nginx on second server
-    nc = str(nginx_dir / "nginx.conf")
-    return _run_shell(("ssh", "-i", ssh_key, remote, "nginx", "-t", "-c", nc))
-
-
-def remote_reload_nginx(remote, ssh_key):
-    return _run_shell(("ssh", "-i", ssh_key, remote, "nginx", "-s", "reload"))

nginx_configurator/templating.py

@@ -1,15 +0,0 @@
-import pkgutil
-from jinja2 import Environment, FunctionLoader, select_autoescape
-
-
-def load_template(name):
-    """
-    Loads file from the templates folder and returns file contents as a string.
-    See jinja2.FunctionLoader docs.
-    """
-    return pkgutil.get_data(f"{__package__}.templates", name).decode("utf-8")
-
-
-jinja = Environment(
-    loader=FunctionLoader(load_template), autoescape=select_autoescape()
-)

requirements.txt

@@ -1,4 +1,3 @@
 click
-python-dotenv
 sysrsync
-Jinja2
+pyyaml

setup.py

@@ -0,0 +1,17 @@
+from setuptools import setup
+
+setup(
+    name='ncc',
+    version='1.0.0',
+    py_modules=['ncc'],
+    install_requires=[
+        'click',
+        'sysrsync',
+        'pyyaml',
+    ],
+    entry_points={
+        'console_scripts': [
+            'ncc = ncc.main:cli',
+        ],
+    },
+)