Changes to make running appyters locally in various ways easy

MANIFEST.in

@@ -8,4 +8,5 @@ recursive-include * *.eot
 recursive-include * *.svg
 recursive-include * *.ttf
 recursive-include * *.woff
-recursive-include * *.woff2
+recursive-include * *.woff2
+recursive-include * VERSION

appyter/VERSION

@@ -0,0 +1 @@
+0.11.0

appyter/__init__.py

@@ -1,3 +1,6 @@
 ''' This module enables you to turn your jupyter notebook into a jinja2 template-driven web application. Or just parse for other purposes.
 '''
+import os
 from appyter.magic import init
+
+__version__ = open(os.path.join(os.path.dirname(__file__), 'VERSION'), 'r').read()

appyter/cli.py

@@ -1,6 +1,11 @@
 import click
+from appyter import __version__
 from click_default_group import DefaultGroup
 
 @click.group(cls=DefaultGroup, default='flask-app')
+@click.version_option(
+  prog_name='appyter',
+  version=__version__,
+)
 def cli():
   pass

appyter/context.py

@@ -4,7 +4,9 @@
 import logging
 logger = logging.getLogger(__name__)
 from appyter.cli import cli
-from appyter.util import importdir_deep, join_routes, try_json_loads
+from appyter.ext.click import click_option_setenv, click_argument_setenv
+from appyter.ext.json import try_json_loads
+from appyter.util import importdir_deep, join_routes
 
 def find_fields_dir_mappings(config=None):
   assert config is not None
@@ -32,7 +34,7 @@ def find_fields(config=None):
   return ctx
 
 @cli.command(help='List the available fields')
-@click.option('--cwd', envvar='APPYTER_CWD', default=os.getcwd(), help='The directory to treat as the current working directory for templates and execution')
+@click_option_setenv('--cwd', envvar='APPYTER_CWD', default=os.getcwd(), help='The directory to treat as the current working directory for templates and execution')
 def list_fields(**kwargs):
   fields = find_fields(get_env(ipynb='app.ipynb', **kwargs))
   field_name_max_size = max(map(len, fields.keys()))
@@ -45,8 +47,8 @@ def list_fields(**kwargs):
     print(field.ljust(field_name_max_size+1), doc_first_line)
 
 @cli.command(help='Describe a field using its docstring')
-@click.option('--cwd', envvar='APPYTER_CWD', default=os.getcwd(), help='The directory to treat as the current working directory for templates and execution')
-@click.argument('field', envvar='APPYTER_FIELD', type=str)
+@click_option_setenv('--cwd', envvar='APPYTER_CWD', default=os.getcwd(), help='The directory to treat as the current working directory for templates and execution')
+@click_argument_setenv('field', envvar='APPYTER_FIELD', type=str)
 def describe_field(field, **kwargs):
   fields = find_fields(get_env(ipynb='app.ipynb', **kwargs))
   assert field in fields, 'Please choose a valid field name, see list-fields for options'

appyter/ext/click/__init__.py

@@ -0,0 +1,42 @@
+import click
+from appyter.ext.json import try_json_loads, try_json_dumps
+
+class Json(click.ParamType):
+  name = 'Json'
+
+  def convert(self, value, param, ctx):
+    return try_json_loads(value)
+
+  def split_envvar_value(self, rv):
+    v = try_json_loads(rv)
+    return v if type(v) == list else [v]
+
+def click_option_setenv(spec, envvar=None, **kwargs):
+  ''' Like click.option but explicitly set os.environ as well.
+  '''
+  import os, re, functools
+  m = re.match(r'^--(.+)$', spec)
+  assert m
+  var = m.group(1).replace('-', '_')
+  def decorator(func):
+    @click.option(spec, envvar=envvar, **kwargs)
+    @functools.wraps(func)
+    def wrapper(**kwargs):
+      if kwargs.get(var) is not None:
+        os.environ[envvar] = try_json_dumps(kwargs[var])
+      return func(**kwargs)
+    return wrapper
+  return decorator
+
+def click_argument_setenv(var, envvar=None, **kwargs):
+  ''' Like click.argument but explicitly set os.environ as well.
+  '''
+  import os, re, functools
+  def decorator(func):
+    @click.argument(var, envvar=envvar, **kwargs)
+    @functools.wraps(func)
+    def wrapper(**kwargs):
+      os.environ[envvar] = try_json_dumps(kwargs[var.replace('-', '_')])
+      return func(**kwargs)
+    return wrapper
+  return decorator

appyter/ext/fs/__init__.py

@@ -45,6 +45,9 @@ def open(self, path, mode='r'):
   def exists(self, path):
     return self._fs.exists(path)
   #
+  def ls(self, path=''):
+    return self._fs.ls(path=path)
+  #
   def cp(self, src, dst):
     return self._fs.cp(src, dst)
   #

appyter/ext/fs/file.py

@@ -1,4 +1,5 @@
 import os
+import glob
 import shutil
 import urllib.parse
 import logging
@@ -42,6 +43,13 @@ def exists(self, path):
       logger.error(traceback.format_exc())
       raise Exception(f"An error occurred while trying to access {path}")
   #
+  def ls(self, path=''):
+    ls_path = FS.join(self._prefix, path) if path else self._prefix
+    return [
+      f[len(ls_path)+1:]
+      for f in glob.glob(ls_path + '/*') + glob.glob(ls_path + '/**/*')
+    ]
+  #
   def cp(self, src, dst):
     try:
       assert src and dst

appyter/ext/fs/s3.py

@@ -4,6 +4,7 @@
 import logging
 logger = logging.getLogger(__name__)
 
+from appyter.ext.fs import Filesystem as FS
 
 class Filesystem:
   def __init__(self, uri, asynchronous=False):
@@ -58,6 +59,13 @@ def exists(self, path):
       logger.error(traceback.format_exc())
       raise Exception(f"An error occurred while trying to access {path}")
   #
+  def ls(self, path=''):
+    ls_path = FS.join(self._prefix, path) if path else self._prefix
+    return [
+      f[len(ls_path)+1:]
+      for f in self._fs.glob(ls_path + '/*') + self._fs.glob(ls_path + '/**/*')
+    ]
+  #
   def cp(self, src, dst):
     try:
       assert src and dst

appyter/ext/json.py

@@ -0,0 +1,13 @@
+import json
+
+def try_json_loads(v):
+  try:
+    return json.loads(v)
+  except:
+    return v
+
+def try_json_dumps(v):
+  if type(v) == str:
+    return v
+  else:
+    return json.dumps(v)

appyter/extras/__init__.py

@@ -9,3 +9,4 @@ def list_extras():
   print('hide-code    Hide code by default')
   print('toc          Add a sticky table of contents to the rendered notebook')
   print('toggle-code  Add a button for toggling code visibility')
+  print('catalog-integration  Add various appyter-catalog-integrated specific features')

appyter/helpers/dockerize.py

@@ -3,11 +3,12 @@
 from appyter.ext.fs import Filesystem
 from appyter.cli import cli
 from appyter.context import get_env, get_jinja2_env
+from appyter.ext.click import click_option_setenv, click_argument_setenv
 
 @cli.command(help='Dockerize an appyter for maximum reproducibility')
-@click.option('--cwd', envvar='APPYTER_CWD', default=os.getcwd(), help='The directory to treat as the current working directory for templates and execution')
-@click.option('--output', envvar='APPYTER_OUTPUT', default='-', type=click.File('w'), help='The output location of the serialized dockerfile')
-@click.argument('ipynb', envvar='APPYTER_IPYNB')
+@click.option('-o', '--output', envvar='APPYTER_OUTPUT', default='-', type=click.File('w'), help='The output location of the serialized dockerfile')
+@click_option_setenv('--cwd', envvar='APPYTER_CWD', default=os.getcwd(), help='The directory to treat as the current working directory for templates and execution')
+@click_argument_setenv('ipynb', envvar='APPYTER_IPYNB')
 def dockerize(ipynb, cwd, output, **kwargs):
   env = get_jinja2_env(
     config=get_env(cwd=cwd, ipynb=ipynb, mode='dockerize', **kwargs),

appyter/helpers/fetch_and_serve.py

@@ -0,0 +1,82 @@
+import os
+import sys
+import click
+from appyter import __version__
+from appyter.cli import cli
+from appyter.ext.click import click_option_setenv, click_argument_setenv
+from appyter.util import join_routes
+
+@cli.command(
+  help='An alias for jupyter notebook which pre-fetches notebook data from a remote appyter and then launches serve',
+  context_settings=dict(
+    ignore_unknown_options=True,
+  ),
+)
+@click_option_setenv('--data-dir', envvar='APPYTER_DATA_DIR', default=None, help='The directory to store data of executions')
+@click_option_setenv('--cwd', envvar='APPYTER_CWD', default=os.getcwd(), help='The directory to treat as the current working directory for templates and execution')
+@click_option_setenv('--host', envvar='APPYTER_HOST', default='0.0.0.0', help='The host to bind to')
+@click_option_setenv('--port', envvar='APPYTER_PORT', type=int, default=5000, help='The port this server should run on')
+@click.argument('args', nargs=-1, type=click.UNPROCESSED)
+@click_argument_setenv('uri', envvar='APPYTER_URI', nargs=1, type=str)
+@click.pass_context
+def fetch_and_serve(ctx, data_dir, cwd, host, port, args, uri):
+  import json
+  import tempfile
+  import urllib.request
+  import nbformat as nbf
+  # fetch the actual notebook
+  try:
+    with urllib.request.urlopen(
+      urllib.request.Request(uri, headers={ 'Accept': 'application/vnd.jupyter' })
+    ) as fr:
+      nb = nbf.read(fr, as_version=4)
+  except:
+    import traceback
+    traceback.print_exc()
+    click.echo('Error fetching appyter instance, is the url right?')
+  #
+  metadata = nb.get('metadata', {}).get('appyter', {})
+  #
+  nbconstruct_version = metadata.get('nbconstruct', {}).get('version', 'unknown')
+  if nbconstruct_version and nbconstruct_version == __version__:
+    pass
+  else:
+    click.echo(f"WARNING: this appyter was not created with this version, instance version was {nbconstruct_version} and our version is {__version__}. Proceed with caution")
+  #
+  nbexecute_version = metadata.get('nbexecute', {}).get('version', 'unknown')
+  if nbexecute_version and nbexecute_version == __version__:
+    pass
+  else:
+    click.echo(f"WARNING: this appyter was not executed with this version, instance version was {nbexecute_version} and our version is {__version__}. Proceed with caution")
+  #
+  if 'nbexecute' not in metadata:
+    click.echo('WARNING: this appyter instance has not been executed, no results will be available')
+  elif 'started' in metadata['nbexecute'] and 'completed' not in metadata['nbexecute']:
+    click.echo('WARNING: this appyter is not finished running, no results will be available')
+  elif 'started' in metadata['nbexecute'] and 'completed' in metadata['nbexecute']:
+    click.echo(f"Appyter ran from {metadata['nbexecute']['started']} to {metadata['nbexecute']['completed']}")
+  else:
+    click.echo('WARNING: this appyter seems old, this may not work properly, please contact us and we can update it')
+  # if tmpdir doesn't exist, create it
+  if data_dir is None:
+    data_dir = tempfile.mkdtemp()
+  # write notebook to data_dir
+  os.makedirs(data_dir, exist_ok=True)
+  filename = metadata.get('nbconstruct', {}).get('filename', 'appyter.ipynb')
+  with open(os.path.join(data_dir, filename), 'w') as fw:
+    nbf.write(nb, fw)
+  #
+  # download all files to data_dir (get files from nbexecute if available otherwise fall back to nbconstruct input-files)
+  files = metadata.get('nbexecute', {}).get('files', metadata.get('nbconstruct', {}).get('files', {}))
+  for file, fileurl in files.items():
+    # relative file paths (those without schemes) are relative to the base uri
+    # TODO: in the future we might be able to mount these paths as we do in the job
+    if '://' not in fileurl:
+      fileurl = join_routes(uri, fileurl)[1:]
+    click.echo(f"Fetching {file} from {fileurl}...")
+    urllib.request.urlretrieve(fileurl, os.path.join(data_dir, file))
+  #
+  click.echo(f"Done. Starting `appyter serve`...")
+  # serve the bundle in jupyter notebook
+  from appyter.helpers.serve import serve
+  ctx.invoke(serve, cwd=cwd, data_dir=data_dir, host=host, port=port, args=[*args, filename])

appyter/helpers/nbclean.py

@@ -4,9 +4,10 @@
 from appyter.ext.fs import Filesystem
 from appyter.cli import cli
 from appyter.context import get_env, get_jinja2_env
+from appyter.ext.click import click_argument_setenv
 
 @cli.command(help='Clean an appyters output & metadata to minimize unnecessary data transfers in production')
-@click.argument('ipynb', type=str, envvar='APPYTER_IPYNB')
+@click_argument_setenv('ipynb', type=str, envvar='APPYTER_IPYNB')
 def nbclean(ipynb, **kwargs):
   nb = nbformat.read(open(ipynb, 'r'), as_version=4)
   for cell in nb.cells:

appyter/helpers/serve.py

@@ -0,0 +1,70 @@
+import os
+import sys
+import click
+from appyter.cli import cli
+from appyter.ext.click import click_option_setenv
+
+@cli.command(
+  help='A simple alias for jupyter notebook which re-uses appyter environment config',
+  context_settings=dict(
+    ignore_unknown_options=True,
+  ),
+)
+@click_option_setenv('--data-dir', envvar='APPYTER_DATA_DIR', default=None, help='The directory to store data of executions')
+@click_option_setenv('--cwd', envvar='APPYTER_CWD', default=os.getcwd(), help='The directory to treat as the current working directory for templates and execution')
+@click_option_setenv('--host', envvar='APPYTER_HOST', default='0.0.0.0', help='The host to bind to')
+@click_option_setenv('--port', envvar='APPYTER_PORT', type=int, default=5000, help='The port this server should run on')
+@click.argument('args', nargs=-1, type=click.UNPROCESSED)
+def serve(data_dir, cwd, host, port, args):
+  import tempfile
+  from subprocess import Popen
+  # add cwd (the appyter itself) to the PYTHONPATH
+  sys.path.insert(0, os.path.realpath(cwd))
+  # create blank data_dir as tmpdir if it doesn't exist
+  if data_dir is None:
+    data_dir = tempfile.mkdtemp()
+  #
+  # but use data_dir (the appyter files) as the cwd of the jupyter notebook
+  os.makedirs(data_dir, exist_ok=True)
+  #
+  # prepare default flags
+  flags = {
+    'NotebookApp.ip': host,
+    'NotebookApp.port': port,
+    'NotebookApp.token': '',
+    'NotebookApp.password': '',
+  }
+  # look through args and replace any overrides
+  extra_args = []
+  for arg in args:
+    if arg.startswith('--'):
+      name, value = arg[2:].split('=', maxsplit=1)
+      flags[name] = value
+    else:
+      extra_args.append(arg)
+  #
+  # run jupyter notebook interactively and exit with it
+  proc = Popen(
+    [
+      'jupyter', 'notebook',
+      *[
+        f"--{key}={value}"
+        for key, value in flags.items()
+      ],
+      *extra_args,
+    ],
+    stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr,
+    cwd=os.path.realpath(data_dir),
+    env=dict(
+      PYTHONPATH=':'.join(sys.path),
+      PATH=os.environ['PATH']
+    ),
+  )
+  exit_code = None
+  while exit_code is None:
+    try:
+      exit_code = proc.wait()
+    except:
+      pass
+  #
+  sys.exit(exit_code)

appyter/orchestration/dispatcher/__init__.py

@@ -3,7 +3,7 @@
 logger = logging.getLogger(__name__)
 
 from appyter.orchestration.cli import orchestration
-from appyter.util import click_option_setenv
+from appyter.ext.click import click_option_setenv
 
 
 def create_app(**kwargs):

appyter/profiles/default/fields/MultiCheckboxField.py

@@ -1,6 +1,5 @@
-import json
 from appyter.fields import Field
-from appyter.util import try_json_loads
+from appyter.ext.json import try_json_loads
 
 class MultiCheckboxField(Field):
   ''' Represing a set of independently selectable check boxes.

appyter/profiles/default/fields/MultiChoiceField.py

@@ -1,5 +1,5 @@
 from appyter.fields import Field
-from appyter.util import try_json_loads
+from appyter.ext.json import try_json_loads
 
 class MultiChoiceField(Field):
   ''' Represing a multi-selectable combo box.